mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-04 17:38:10 +00:00
REMOVE DEPRECATED CODE/FILES, SUCH AS VOICE CONVERSION
This commit is contained in:
Brandon Thomas committed
1 parent
ad23fe0c80
commit
2eabd03b48
45 files changed
-5903
No files matched your search
Generated
-728
File diff suppressed because it is too large.
Load diff
@@ -6,16 +6,10 @@ authors = [ "Brandon Thomas <bt@brand.io>",
|
||||
|
||||
[workspace]
|
||||
members = [
|
||||
"tacotron_melgan",
|
||||
"tacotron_tensorflow",
|
||||
"tts_service",
|
||||
"voice_converter_sidecar",
|
||||
]
|
||||
|
||||
default-members = [
|
||||
"tacotron_melgan",
|
||||
"tacotron_tensorflow",
|
||||
"tts_service",
|
||||
"voice_converter_sidecar",
|
||||
]
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
-153
@@ -1,153 +0,0 @@
|
||||
---
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: tts
|
||||
namespace: mumble
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
selector:
|
||||
app: tts
|
||||
ports:
|
||||
- name: http
|
||||
protocol: TCP
|
||||
port: 80
|
||||
targetPort: 12345
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: tts
|
||||
namespace: mumble
|
||||
labels:
|
||||
app: tts
|
||||
spec:
|
||||
replicas: 3 # 4 would never launch with cluster configs as of 2020-06-20
|
||||
selector:
|
||||
matchLabels:
|
||||
app: tts
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: tts
|
||||
annotations:
|
||||
bump: '1'
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 60
|
||||
imagePullSecrets:
|
||||
- name: regcred
|
||||
volumes:
|
||||
- name: downloads
|
||||
hostPath:
|
||||
path: /downloads
|
||||
type: DirectoryOrCreate
|
||||
containers:
|
||||
- name: downloader
|
||||
image: docker.pkg.github.com/ml-applications/do-spaces-downloader/do-spaces-downloader:58ec25e3aa3c
|
||||
command: ["./launch.sh"]
|
||||
args: [""]
|
||||
#command: ["sleep"]
|
||||
#args: ["300000"]
|
||||
volumeMounts:
|
||||
- name: downloads
|
||||
mountPath: /data/downloads
|
||||
env:
|
||||
- name: ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: spaces-secrets
|
||||
key: aws_access_key_id
|
||||
- name: SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: spaces-secrets
|
||||
key: aws_secret_access_key
|
||||
- name: BUCKET_NAME
|
||||
value: "upload-models"
|
||||
- name: REGION
|
||||
value: "nyc3"
|
||||
# NB: These filesystem paths are strage because they need to be on the
|
||||
# same filesystem. Rust can't handle moving files between filesystems yet:
|
||||
# "Error: Invalid cross-device link (os error 18)"
|
||||
- name: DOWNLOAD_DIR
|
||||
value: "/data/downloads/downloaded"
|
||||
- name: TEMP_DIR
|
||||
value: "/data/downloads/tmp"
|
||||
- name: MATCH_PATH
|
||||
value: "" # previously "models/"
|
||||
- name: SLEEP_TIMEOUT
|
||||
value: "300"
|
||||
- name: RUST_LOG
|
||||
value: "info,do_spaces_downloader=debug"
|
||||
- name: tts
|
||||
# NB: Container builds are triggered by Github Actions
|
||||
|
||||
# The following image runs a non-Arpabet engine that can run an
|
||||
# LSpeech model that sounds 100% amazing
|
||||
# image: docker.pkg.github.com/echelon/voder/tts-service:6b5c22567614
|
||||
#
|
||||
image: docker.pkg.github.com/ml-applications/voder/tts-service:60211d6df69a
|
||||
resources:
|
||||
limits:
|
||||
# 4 GiB (4096 MiB)
|
||||
# 3 GiB (3072 MiB)
|
||||
# 2 GiB (2048 MiB)
|
||||
memory: "4Gi" # Just under 4 GB
|
||||
cpu: "700m" # 700 milliCPU
|
||||
requests:
|
||||
#memory: "50Mi" - we're getting evicted!
|
||||
# Warning FailedScheduling <unknown> default-scheduler 0/6 nodes are available: 6 Insufficient memory.
|
||||
# Normal NotTriggerScaleUp 29s cluster-autoscaler pod didn't trigger scale-up (it wouldn't fit if
|
||||
# a new node is added):
|
||||
#
|
||||
memory: "4Gi" # Just under 4 GB
|
||||
cpu: "700m" # 700 milliCPU
|
||||
volumeMounts:
|
||||
- name: downloads
|
||||
mountPath: /data/downloads
|
||||
env:
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: db-secrets
|
||||
key: url
|
||||
- name: BIND_ADDRESS
|
||||
value: "0.0.0.0:12345"
|
||||
- name: NUM_WORKERS
|
||||
value: "5"
|
||||
- name: ASSET_DIRECTORY
|
||||
value: "/data/downloads/downloaded/frontend/build" # TODO
|
||||
- name: MODEL_CONFIG_FILE
|
||||
value: "/data/downloads/downloaded/models/deployed_models.toml"
|
||||
- name: MAX_CHAR_LEN
|
||||
value: "220"
|
||||
command: ["./launch.sh"]
|
||||
args: [""]
|
||||
#command: ["sleep"]
|
||||
#args: ["300000"]
|
||||
ports:
|
||||
- containerPort: 12345
|
||||
protocol: TCP
|
||||
# If the readiness probe fails, the container is removed from the service.
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readiness
|
||||
port: 12345
|
||||
# Check every two seconds.
|
||||
periodSeconds: 2
|
||||
# Warmup period before first check.
|
||||
initialDelaySeconds: 5
|
||||
# How long it takes for the probe to time out.
|
||||
timeoutSeconds: 1
|
||||
# Number of attempts before the pod is marked unready
|
||||
failureThreshold: 1
|
||||
# If the liveness probe fails, the container is subject to its restart policy
|
||||
# Since the container can crash on its own, we'll omit this.
|
||||
# livenessProbe:
|
||||
# httpGet:
|
||||
# path: /liveness
|
||||
# port: 12345
|
||||
# initialDelaySeconds: 20
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 3
|
||||
# failureThreshold: 3
|
||||
@@ -1,358 +0,0 @@
|
||||
---
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: service-trump
|
||||
namespace: vocodes
|
||||
spec:
|
||||
#type: LoadBalancer
|
||||
#type: ClusterIP
|
||||
selector:
|
||||
#model: trump
|
||||
app: tts-trump
|
||||
ports:
|
||||
- name: http
|
||||
protocol: TCP
|
||||
port: 12345
|
||||
targetPort: 12345
|
||||
|
||||
---
|
||||
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: service-other
|
||||
namespace: vocodes
|
||||
spec:
|
||||
#type: LoadBalancer
|
||||
selector:
|
||||
model: other
|
||||
ports:
|
||||
- name: http
|
||||
protocol: TCP
|
||||
port: 80
|
||||
targetPort: 12345
|
||||
---
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: tts-service-proxy
|
||||
namespace: vocodes
|
||||
labels:
|
||||
app: tts-service-proxy
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: tts-service-proxy
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: tts-service-proxy
|
||||
annotations:
|
||||
bump: '2'
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 60
|
||||
imagePullSecrets:
|
||||
- name: regcred
|
||||
containers:
|
||||
- name: tts-service-proxy
|
||||
image: docker.pkg.github.com/ml-applications/voder-proxy/tts-service-proxy:67d094d38ef4
|
||||
#command: ["./launch.sh"]
|
||||
#args: [""]
|
||||
#command: ["sleep"]
|
||||
#args: ["300000"]
|
||||
env:
|
||||
- name: ROUTE_ONE
|
||||
value: "http://10.245.22.66:80"
|
||||
- name: ROUTE_TWO
|
||||
value: "http://10.245.179.52:80"
|
||||
- name: RUST_LOG
|
||||
value: "info"
|
||||
ports:
|
||||
- containerPort: 13900
|
||||
protocol: TCP
|
||||
|
||||
---
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: tts-trump
|
||||
namespace: vocodes
|
||||
labels:
|
||||
app: tts-trump
|
||||
#model: trump
|
||||
spec:
|
||||
replicas: 1 # 4 would never launch with cluster configs as of 2020-06-20
|
||||
selector:
|
||||
matchLabels:
|
||||
app: tts-trump
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: tts-trump
|
||||
annotations:
|
||||
bump: '1'
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 60
|
||||
imagePullSecrets:
|
||||
- name: regcred
|
||||
volumes:
|
||||
- name: downloads
|
||||
hostPath:
|
||||
path: /downloads
|
||||
type: DirectoryOrCreate
|
||||
containers:
|
||||
- name: downloader
|
||||
image: docker.pkg.github.com/ml-applications/do-spaces-downloader/do-spaces-downloader:58ec25e3aa3c
|
||||
command: ["./launch.sh"]
|
||||
args: [""]
|
||||
#command: ["sleep"]
|
||||
#args: ["300000"]
|
||||
volumeMounts:
|
||||
- name: downloads
|
||||
mountPath: /data/downloads
|
||||
env:
|
||||
- name: ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: spaces-secrets
|
||||
key: aws_access_key_id
|
||||
- name: SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: spaces-secrets
|
||||
key: aws_secret_access_key
|
||||
- name: BUCKET_NAME
|
||||
value: "upload-models"
|
||||
- name: REGION
|
||||
value: "nyc3"
|
||||
# NB: These filesystem paths are strage because they need to be on the
|
||||
# same filesystem. Rust can't handle moving files between filesystems yet:
|
||||
# "Error: Invalid cross-device link (os error 18)"
|
||||
- name: DOWNLOAD_DIR
|
||||
value: "/data/downloads/downloaded"
|
||||
- name: TEMP_DIR
|
||||
value: "/data/downloads/tmp"
|
||||
- name: MATCH_PATH
|
||||
value: "" # previously "models/"
|
||||
- name: SLEEP_TIMEOUT
|
||||
value: "300"
|
||||
- name: RUST_LOG
|
||||
value: "info,do_spaces_downloader=debug"
|
||||
- name: tts
|
||||
# NB: Container builds are triggered by Github Actions
|
||||
|
||||
# The following image runs a non-Arpabet engine that can run an
|
||||
# LSpeech model that sounds 100% amazing
|
||||
# image: docker.pkg.github.com/echelon/voder/tts-service:6b5c22567614
|
||||
#
|
||||
image: docker.pkg.github.com/ml-applications/voder/tts-service:60211d6df69a
|
||||
resources:
|
||||
limits:
|
||||
# 4 GiB (4096 MiB)
|
||||
# 3 GiB (3072 MiB)
|
||||
# 2 GiB (2048 MiB)
|
||||
memory: "4Gi" # Just under 4 GB
|
||||
cpu: "700m" # 700 milliCPU
|
||||
requests:
|
||||
#memory: "50Mi" - we're getting evicted!
|
||||
# Warning FailedScheduling <unknown> default-scheduler 0/6 nodes are available: 6 Insufficient memory.
|
||||
# Normal NotTriggerScaleUp 29s cluster-autoscaler pod didn't trigger scale-up (it wouldn't fit if
|
||||
# a new node is added):
|
||||
#
|
||||
memory: "2Gi" # Just under 4 GB
|
||||
cpu: "300m" # 700 milliCPU
|
||||
volumeMounts:
|
||||
- name: downloads
|
||||
mountPath: /data/downloads
|
||||
env:
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: db-secrets
|
||||
key: url
|
||||
- name: BIND_ADDRESS
|
||||
value: "0.0.0.0:12345"
|
||||
- name: NUM_WORKERS
|
||||
value: "5"
|
||||
- name: ASSET_DIRECTORY
|
||||
value: "/data/downloads/downloaded/frontend/build" # TODO
|
||||
- name: MODEL_CONFIG_FILE
|
||||
value: "/data/downloads/downloaded/models/deployed_models.toml"
|
||||
- name: MAX_CHAR_LEN
|
||||
value: "220"
|
||||
command: ["./launch.sh"]
|
||||
args: [""]
|
||||
#command: ["sleep"]
|
||||
#args: ["300000"]
|
||||
ports:
|
||||
- containerPort: 12345
|
||||
protocol: TCP
|
||||
# If the readiness probe fails, the container is removed from the service.
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readiness
|
||||
port: 12345
|
||||
# Check every two seconds.
|
||||
periodSeconds: 2
|
||||
# Warmup period before first check.
|
||||
initialDelaySeconds: 5
|
||||
# How long it takes for the probe to time out.
|
||||
timeoutSeconds: 1
|
||||
# Number of attempts before the pod is marked unready
|
||||
failureThreshold: 1
|
||||
# If the liveness probe fails, the container is subject to its restart policy
|
||||
# Since the container can crash on its own, we'll omit this.
|
||||
# livenessProbe:
|
||||
# httpGet:
|
||||
# path: /liveness
|
||||
# port: 12345
|
||||
# initialDelaySeconds: 20
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 3
|
||||
# failureThreshold: 3
|
||||
#
|
||||
|
||||
---
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: tts-other
|
||||
namespace: vocodes
|
||||
labels:
|
||||
app: tts-other
|
||||
model: other
|
||||
spec:
|
||||
replicas: 1 # 4 would never launch with cluster configs as of 2020-06-20
|
||||
selector:
|
||||
matchLabels:
|
||||
app: tts-other
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: tts-other
|
||||
annotations:
|
||||
bump: '1'
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 60
|
||||
imagePullSecrets:
|
||||
- name: regcred
|
||||
volumes:
|
||||
- name: downloads
|
||||
hostPath:
|
||||
path: /downloads
|
||||
type: DirectoryOrCreate
|
||||
containers:
|
||||
- name: downloader
|
||||
image: docker.pkg.github.com/ml-applications/do-spaces-downloader/do-spaces-downloader:58ec25e3aa3c
|
||||
command: ["./launch.sh"]
|
||||
args: [""]
|
||||
#command: ["sleep"]
|
||||
#args: ["300000"]
|
||||
volumeMounts:
|
||||
- name: downloads
|
||||
mountPath: /data/downloads
|
||||
env:
|
||||
- name: ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: spaces-secrets
|
||||
key: aws_access_key_id
|
||||
- name: SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: spaces-secrets
|
||||
key: aws_secret_access_key
|
||||
- name: BUCKET_NAME
|
||||
value: "upload-models"
|
||||
- name: REGION
|
||||
value: "nyc3"
|
||||
# NB: These filesystem paths are strage because they need to be on the
|
||||
# same filesystem. Rust can't handle moving files between filesystems yet:
|
||||
# "Error: Invalid cross-device link (os error 18)"
|
||||
- name: DOWNLOAD_DIR
|
||||
value: "/data/downloads/downloaded"
|
||||
- name: TEMP_DIR
|
||||
value: "/data/downloads/tmp"
|
||||
- name: MATCH_PATH
|
||||
value: "" # previously "models/"
|
||||
- name: SLEEP_TIMEOUT
|
||||
value: "300"
|
||||
- name: RUST_LOG
|
||||
value: "info,do_spaces_downloader=debug"
|
||||
- name: tts
|
||||
# NB: Container builds are triggered by Github Actions
|
||||
|
||||
# The following image runs a non-Arpabet engine that can run an
|
||||
# LSpeech model that sounds 100% amazing
|
||||
# image: docker.pkg.github.com/echelon/voder/tts-service:6b5c22567614
|
||||
#
|
||||
image: docker.pkg.github.com/ml-applications/voder/tts-service:60211d6df69a
|
||||
resources:
|
||||
limits:
|
||||
# 4 GiB (4096 MiB)
|
||||
# 3 GiB (3072 MiB)
|
||||
# 2 GiB (2048 MiB)
|
||||
memory: "4Gi" # Just under 4 GB
|
||||
cpu: "700m" # 700 milliCPU
|
||||
requests:
|
||||
#memory: "50Mi" - we're getting evicted!
|
||||
# Warning FailedScheduling <unknown> default-scheduler 0/6 nodes are available: 6 Insufficient memory.
|
||||
# Normal NotTriggerScaleUp 29s cluster-autoscaler pod didn't trigger scale-up (it wouldn't fit if
|
||||
# a new node is added):
|
||||
#
|
||||
memory: "2Gi" # Just under 4 GB
|
||||
cpu: "300m" # 700 milliCPU
|
||||
volumeMounts:
|
||||
- name: downloads
|
||||
mountPath: /data/downloads
|
||||
env:
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: db-secrets
|
||||
key: url
|
||||
- name: BIND_ADDRESS
|
||||
value: "0.0.0.0:12345"
|
||||
- name: NUM_WORKERS
|
||||
value: "5"
|
||||
- name: ASSET_DIRECTORY
|
||||
value: "/data/downloads/downloaded/frontend/build" # TODO
|
||||
- name: MODEL_CONFIG_FILE
|
||||
value: "/data/downloads/downloaded/models/deployed_models.toml"
|
||||
- name: MAX_CHAR_LEN
|
||||
value: "220"
|
||||
command: ["./launch.sh"]
|
||||
args: [""]
|
||||
#command: ["sleep"]
|
||||
#args: ["300000"]
|
||||
ports:
|
||||
- containerPort: 12345
|
||||
protocol: TCP
|
||||
# If the readiness probe fails, the container is removed from the service.
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readiness
|
||||
port: 12345
|
||||
# Check every two seconds.
|
||||
periodSeconds: 2
|
||||
# Warmup period before first check.
|
||||
initialDelaySeconds: 5
|
||||
# How long it takes for the probe to time out.
|
||||
timeoutSeconds: 1
|
||||
# Number of attempts before the pod is marked unready
|
||||
failureThreshold: 1
|
||||
# If the liveness probe fails, the container is subject to its restart policy
|
||||
# Since the container can crash on its own, we'll omit this.
|
||||
# livenessProbe:
|
||||
# httpGet:
|
||||
# path: /liveness
|
||||
# port: 12345
|
||||
# initialDelaySeconds: 20
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 3
|
||||
# failureThreshold: 3
|
||||
@@ -1,35 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# For some reason, pods aren't going away after they're marked for deletion.
|
||||
# They're getting stuck as "Terminating" for days at a time, leading to an
|
||||
# accumulation of limitless stale pods.
|
||||
#
|
||||
# I'm not sure what the cause is, but there are lots of reports of this on
|
||||
# Google.
|
||||
#
|
||||
# One thing I uncovered (in kubernetes events) that might be related:
|
||||
#
|
||||
# 22m Normal TaintManagerEviction pod/{name} Marking for deletion Pod {name}
|
||||
#
|
||||
# But I honestly don't know what is causing this. Perhaps the fuse filesytem?
|
||||
# Perhaps DigitalOcean weirdness?
|
||||
#
|
||||
# This stop-gap non-solution is taken from https://stackoverflow.com/q/50336665
|
||||
#
|
||||
function cleanup_pods() {
|
||||
namespace=$1
|
||||
|
||||
echo "Cleanup pods in namespace ${namespace}"
|
||||
|
||||
pods=$( kubectl -n ${namespace} get pods | grep -v Running | tail -n +2 | awk -F " " '{print $1}' )
|
||||
|
||||
for pod in $pods;
|
||||
do
|
||||
echo "Deleting pod ${pod}"
|
||||
kubectl -n ${namespace} delete pod $pod --grace-period=0 --force 2> /dev/null
|
||||
done
|
||||
}
|
||||
|
||||
cleanup_pods "mumble"
|
||||
cleanup_pods "trumped"
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
[package]
|
||||
name = "tacotron_melgan"
|
||||
version = "0.0.1"
|
||||
authors = [ "Brandon Thomas <bt@brand.io>",
|
||||
"Brandon Thomas <echelon@gmail.com>" ]
|
||||
|
||||
[[bin]]
|
||||
name = "tacotron_melgan"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
hound = "3.4.0" # wav codec
|
||||
rand = "0.7.2" # random sampling for testing (TODO: temp)
|
||||
#rocket = "0.4.2" # http server
|
||||
#rocket_contrib = "0.4.2" # http server
|
||||
serde = "1.0.104"
|
||||
serde_derive = "1.0.104"
|
||||
serde_json = "1.0.44"
|
||||
tch = "0.1.3" # libtorch bindings (pytorch)
|
||||
@@ -1,16 +0,0 @@
|
||||
Tacotron + Melgan
|
||||
=================
|
||||
Runs Tacotron2 + Melgan as a fully end-to-end TTS system.
|
||||
|
||||
Compiling
|
||||
---------
|
||||
Libtorch has been problematic, so I have a good version here:
|
||||
|
||||
```
|
||||
export LIBTORCH=/home/bt/dev/3rd/libtorch
|
||||
export LD_LIBRARY_PATH=${LIBTORCH}/lib:$LD_LIBRARY_PATH
|
||||
```
|
||||
|
||||
This was downloaded from
|
||||
https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-1.3.1%2Bcpu.zip
|
||||
|
||||
Whitespace-only changes.
@@ -1,82 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# parts taken from: https://github.com/pytorch/pytorch/issues/20356#issuecomment-545572400
|
||||
"""
|
||||
SUPER STUPID HACK
|
||||
|
||||
libtensor/tch.rs do not seem to be able to load tensors saved from
|
||||
mytorch with `foo_tensor.save()`. This gets around the problem by
|
||||
wrapping tensors in JIT containers that can be loaded. The source
|
||||
tensor can be pulled out by calling `forward` on the module.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
|
||||
# If set, change the tensor type.
|
||||
output_type = 'double'
|
||||
|
||||
if output_type == 'float':
|
||||
torch.set_default_tensor_type('torch.FloatTensor')
|
||||
elif output_type == 'double':
|
||||
torch.set_default_tensor_type('torch.DoubleTensor')
|
||||
|
||||
def cuda_to_cpu(model):
|
||||
"""Recursively make everything non-CUDA"""
|
||||
if isinstance(model, dict) or isinstance(model, OrderedDict):
|
||||
for key, value in model.items():
|
||||
model[key] = cuda_to_cpu(value)
|
||||
return model
|
||||
if isinstance(model, list):
|
||||
for i, value in enumerate(model):
|
||||
model[i] = cuda_to_cpu(value)
|
||||
elif isinstance(model, torch.Tensor):
|
||||
return model.cpu()
|
||||
else:
|
||||
return model
|
||||
|
||||
class Container(torch.nn.Module):
|
||||
def __init__(self, payload):
|
||||
super(Container, self).__init__()
|
||||
self.tensor = payload
|
||||
def forward(self, x):
|
||||
return self.tensor # XXX: This is how we return the wrapped tensor. Such a hack.
|
||||
|
||||
def main(filename):
|
||||
output_filename = '{}.containerized.pt'.format(filename)
|
||||
|
||||
print('Loading tensor file: {}'.format(filename))
|
||||
tensor = torch.load(filename, map_location=torch.device('cpu'))
|
||||
|
||||
if output_type == 'float':
|
||||
tensor = tensor.float()
|
||||
elif output_type == 'double':
|
||||
tensor = tensor.double()
|
||||
|
||||
if not isinstance(tensor, torch.Tensor):
|
||||
type_ = type(tensor)
|
||||
raise Exception('File should contain object of type Tensor, not {}'.format(type_))
|
||||
|
||||
print('Switching device from CUDA to CPU (if necessary)')
|
||||
tensor = cuda_to_cpu(tensor)
|
||||
|
||||
print('Containerizing. Libtorch can\'t read raw tensors from ' \
|
||||
+ 'pytorch (at least I don\'t know how)')
|
||||
|
||||
container = Container(tensor)
|
||||
jit_container = torch.jit.script(container)
|
||||
|
||||
print('Saving: {}'.format(output_filename))
|
||||
jit_container.save(output_filename)
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2:
|
||||
print('Must supply input filename')
|
||||
sys.exit(1)
|
||||
|
||||
filename = sys.argv[1]
|
||||
main(filename)
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
import torch
|
||||
#from librosa.filters import mel as librosa_mel_fn
|
||||
#from audio_processing import dynamic_range_compression
|
||||
#from audio_processing import dynamic_range_decompression
|
||||
#from stft import STFT
|
||||
|
||||
|
||||
class LinearNorm(torch.nn.Module):
|
||||
def __init__(self, in_dim, out_dim, bias=True, w_init_gain='linear'):
|
||||
super(LinearNorm, self).__init__()
|
||||
self.linear_layer = torch.nn.Linear(in_dim, out_dim, bias=bias)
|
||||
|
||||
torch.nn.init.xavier_uniform_(
|
||||
self.linear_layer.weight,
|
||||
gain=torch.nn.init.calculate_gain(w_init_gain))
|
||||
|
||||
def forward(self, x):
|
||||
return self.linear_layer(x)
|
||||
|
||||
|
||||
class ConvNorm(torch.nn.Module):
|
||||
def __init__(self, in_channels, out_channels, kernel_size=1, stride=1,
|
||||
padding=None, dilation=1, bias=True, w_init_gain='linear'):
|
||||
super(ConvNorm, self).__init__()
|
||||
if padding is None:
|
||||
assert(kernel_size % 2 == 1)
|
||||
padding = int(dilation * (kernel_size - 1) / 2)
|
||||
|
||||
self.conv = torch.nn.Conv1d(in_channels, out_channels,
|
||||
kernel_size=kernel_size, stride=stride,
|
||||
padding=padding, dilation=dilation,
|
||||
bias=bias)
|
||||
|
||||
torch.nn.init.xavier_uniform_(
|
||||
self.conv.weight, gain=torch.nn.init.calculate_gain(w_init_gain))
|
||||
|
||||
def forward(self, signal):
|
||||
conv_signal = self.conv(signal)
|
||||
return conv_signal
|
||||
|
||||
|
||||
class TacotronSTFT(torch.nn.Module):
|
||||
def __init__(self, filter_length=1024, hop_length=256, win_length=1024,
|
||||
n_mel_channels=80, sampling_rate=22050, mel_fmin=0.0,
|
||||
mel_fmax=8000.0):
|
||||
super(TacotronSTFT, self).__init__()
|
||||
self.n_mel_channels = n_mel_channels
|
||||
self.sampling_rate = sampling_rate
|
||||
self.stft_fn = STFT(filter_length, hop_length, win_length)
|
||||
mel_basis = librosa_mel_fn(
|
||||
sampling_rate, filter_length, n_mel_channels, mel_fmin, mel_fmax)
|
||||
mel_basis = torch.from_numpy(mel_basis).float()
|
||||
self.register_buffer('mel_basis', mel_basis)
|
||||
|
||||
def spectral_normalize(self, magnitudes):
|
||||
output = dynamic_range_compression(magnitudes)
|
||||
return output
|
||||
|
||||
@torch.jit.ignore
|
||||
def spectral_de_normalize(self, magnitudes):
|
||||
output = dynamic_range_decompression(magnitudes)
|
||||
return output
|
||||
|
||||
def mel_spectrogram(self, y):
|
||||
"""Computes mel-spectrograms from a batch of waves
|
||||
PARAMS
|
||||
------
|
||||
y: Variable(torch.FloatTensor) with shape (B, T) in range [-1, 1]
|
||||
|
||||
RETURNS
|
||||
-------
|
||||
mel_output: torch.FloatTensor of shape (B, n_mel_channels, T)
|
||||
"""
|
||||
assert(torch.min(y.data) >= -1)
|
||||
assert(torch.max(y.data) <= 1)
|
||||
|
||||
magnitudes, phases = self.stft_fn.transform(y)
|
||||
magnitudes = magnitudes.data
|
||||
mel_output = torch.matmul(self.mel_basis, magnitudes)
|
||||
mel_output = self.spectral_normalize(mel_output)
|
||||
return mel_output
|
||||
@@ -1,2 +0,0 @@
|
||||
PyYAML==5.2
|
||||
torch==1.3.1
|
||||
@@ -1,68 +0,0 @@
|
||||
#![feature(proc_macro_hygiene, decl_macro)]
|
||||
|
||||
//#[macro_use] extern crate rocket;
|
||||
//#[macro_use] extern crate rocket_contrib;
|
||||
#[macro_use] extern crate serde_derive;
|
||||
extern crate hound;
|
||||
extern crate rand;
|
||||
extern crate serde;
|
||||
extern crate serde_json;
|
||||
extern crate tch;
|
||||
|
||||
use tch::CModule;
|
||||
use tch::Tensor;
|
||||
use tch::nn::Module;
|
||||
use tch::nn::ModuleT;
|
||||
//use server::run_server;
|
||||
|
||||
pub mod melgan;
|
||||
pub mod model;
|
||||
//pub mod server;
|
||||
pub mod tacotron;
|
||||
|
||||
pub fn main() {
|
||||
println!("Tacotron2 + MelGan");
|
||||
|
||||
//melgan::run_melgan_network();
|
||||
//tacotron::run_tacotron();
|
||||
//run_end_to_end();
|
||||
|
||||
//run_server();
|
||||
}
|
||||
|
||||
fn run_end_to_end() {
|
||||
println!("Load Tacotron Model");
|
||||
let tacotron_model =
|
||||
model::load_model_file("/home/bt/dev/voder/tacotron_melgan/tacotron_jit_model_voder_c0cac635.pt");
|
||||
|
||||
println!("Load Melgan Model");
|
||||
let melgan_model =
|
||||
model::load_model_file("/home/bt/dev/voder/tacotron_melgan/melgan_jit_model_voder_c0cac635.pt");
|
||||
|
||||
println!("Load Text Sequence Tensor");
|
||||
let text_sequence =
|
||||
model::load_wrapped_tensor_file("/home/bt/dev/voder/data/text/tacotron_text_sequence.pt.containerized.pt");
|
||||
|
||||
//for i in 0..10 {
|
||||
println!(">>> Evaluating Tacotron Model");
|
||||
let mut mel_tensor = tacotron_model.forward(&text_sequence);
|
||||
|
||||
println!("Result mel tensor: {:?}", mel_tensor);
|
||||
|
||||
if mel_tensor.dim() == 2 {
|
||||
println!("mel unsqeeze");
|
||||
mel_tensor = mel_tensor.unsqueeze(0);
|
||||
}
|
||||
|
||||
let audio_tensor = melgan_model.forward(&mel_tensor);
|
||||
|
||||
println!("Result audio tensor: {:?}", audio_tensor);
|
||||
|
||||
let audio = melgan::audio_tensor_to_audio_signal(audio_tensor);
|
||||
|
||||
println!("Writing audio file...");
|
||||
melgan::write_audio_file(audio, "double_model_results_2.wav");
|
||||
|
||||
println!("Done.");
|
||||
//}
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
use tch::CModule;
|
||||
use tch::Tensor;
|
||||
use tch::nn::Module;
|
||||
use tch;
|
||||
|
||||
use rand::Rng;
|
||||
|
||||
use model::{
|
||||
load_model_file,
|
||||
load_wrapped_tensor_file,
|
||||
};
|
||||
use hound::Sample;
|
||||
|
||||
const WRAPPED_MODEL_PATH : &'static str = "/home/bt/dev/voder/tacotron_melgan/melgan_jit_model_voder_c0cac635.pt";
|
||||
|
||||
const EXAMPLE_MEL_1: &'static str = "/home/bt/dev/voder/data/mels/LJ002-0320.mel.containerized.pt";
|
||||
const EXAMPLE_MEL_2 : &'static str = "/home/bt/dev/voder/data/mels/trump_2018_02_15-001.mel.containerized.pt";
|
||||
|
||||
//pub const MAX_WAV_VALUE : f32 = 32768.0f32;
|
||||
//pub const HALF_WAV_VALUE : f32 = MAX_WAV_VALUE * 0.5f32;
|
||||
|
||||
// TODO: This is an hparam and should be dynamic.
|
||||
//pub const HOP_LENGTH : i64 = 256;
|
||||
|
||||
pub fn audio_tensor_to_audio_signal(mel: Tensor) -> Vec<i16> {
|
||||
let mut flat_audio_tensor = mel.squeeze();
|
||||
|
||||
println!("Sqeueezed tensor: {:?}, dim: {}",
|
||||
flat_audio_tensor,
|
||||
flat_audio_tensor.dim());
|
||||
|
||||
let length = flat_audio_tensor.size1().unwrap() as usize;
|
||||
println!("Length: {}", length);
|
||||
|
||||
//let trim_back = HOP_LENGTH * 10;
|
||||
//let new_size = length as i64 - trim_back;
|
||||
//println!("Old size: {}", length);
|
||||
//println!("New size: {}", new_size);
|
||||
//flat_audio_tensor = flat_audio_tensor.resize_(&[new_size]);
|
||||
|
||||
//flat_audio_tensor = flat_audio_tensor * MAX_WAV_VALUE as f64;
|
||||
|
||||
let mut data : Vec<f32> = Vec::with_capacity(length);
|
||||
|
||||
for i in 0 .. length {
|
||||
data.push(0.0f32);
|
||||
}
|
||||
|
||||
flat_audio_tensor.copy_data(data.as_mut_slice(), length as usize);
|
||||
|
||||
data.iter().map(|x| x.trunc() as i16).collect()
|
||||
}
|
||||
|
||||
fn debug_print_sample(audio: &Vec<f32>, num_samples: usize) {
|
||||
let mut rng = rand::thread_rng();
|
||||
for _ in 0..num_samples {
|
||||
let ri = rng.gen_range(0, audio.len());
|
||||
println!("audio_data[{}]: {:?}", ri, audio.get(ri as usize));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run_melgan_network() {
|
||||
let mut mel = load_wrapped_tensor_file(EXAMPLE_MEL_1);
|
||||
println!("Got mel: {:?} of dim {}", mel, mel.dim());
|
||||
|
||||
if mel.dim() == 2 {
|
||||
println!("mel unsqeeze");
|
||||
mel = mel.unsqueeze(0);
|
||||
}
|
||||
|
||||
let melgan_model = load_model_file(WRAPPED_MODEL_PATH);
|
||||
|
||||
println!("Evaluating model...");
|
||||
let output = melgan_model.forward(&mel);
|
||||
|
||||
println!("Result tensor: {:?}", output);
|
||||
|
||||
let audio = audio_tensor_to_audio_signal(output);
|
||||
|
||||
//debug_print_sample(&audio, 10);
|
||||
|
||||
write_audio_file(audio, "melgan_output.wav");
|
||||
}
|
||||
|
||||
pub fn write_audio_file(audio_signal: Vec<i16>, filename: &str) {
|
||||
let spec = hound::WavSpec {
|
||||
channels: 1,
|
||||
sample_rate: 20000,
|
||||
bits_per_sample: 16,
|
||||
sample_format: hound::SampleFormat::Int,
|
||||
};
|
||||
|
||||
let mut writer = hound::WavWriter::create(filename, spec).unwrap();
|
||||
|
||||
for sample in audio_signal {
|
||||
//let sample = sample * 0.0001f32;
|
||||
writer.write_sample(sample).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
use hound::SampleFormat;
|
||||
use hound::WavSpec;
|
||||
use hound::WavWriter;
|
||||
|
||||
use tch::{CModule, Device, Kind};
|
||||
use tch::Tensor;
|
||||
use tch::nn::Module;
|
||||
use tch;
|
||||
|
||||
use std::io::Cursor;
|
||||
use std::io::BufWriter;
|
||||
|
||||
use melgan::audio_tensor_to_audio_signal;
|
||||
|
||||
/// Load a Torch model
|
||||
pub fn load_model_file(filename: &str) -> CModule {
|
||||
println!("Loading model: {}", filename);
|
||||
tch::CModule::load(filename).unwrap()
|
||||
}
|
||||
|
||||
/// This is a _HACK_ to load a single Tensor.
|
||||
///
|
||||
/// It seems currently impossible to load tensors into tch.rs (libtorch),
|
||||
/// (eg with 'Tensor::load(filename)') that are saved from pytorch. They
|
||||
/// use different serialization formats. Luckily, I can embed a tensor in
|
||||
/// a JIT module and unpack it from there instead. The most
|
||||
/// straightforward way of getting it out is to define a 'forward()'
|
||||
/// method that simply returns the wrapped tensor.
|
||||
///
|
||||
/// I wrote a script to wrap tensors: 'jit_containerize_tensor.py'.
|
||||
pub fn load_wrapped_tensor_file(filename: &str) -> Tensor {
|
||||
println!("Loading wrapped tensor file: {}", filename);
|
||||
let module = tch::CModule::load(filename).unwrap();
|
||||
let mut temp = Tensor::zeros(
|
||||
&[10, 10, 10],
|
||||
(tch::Kind::Float, tch::Device::Cpu)
|
||||
);
|
||||
module.forward(&temp)
|
||||
}
|
||||
|
||||
pub struct TacoMelModel {
|
||||
tacotron_model: CModule,
|
||||
melgan_model: CModule,
|
||||
}
|
||||
|
||||
impl TacoMelModel {
|
||||
|
||||
pub fn create(tacotron_filename: &str, melgan_filename: &str) -> Self {
|
||||
let tacotron_model = load_model_file(tacotron_filename);
|
||||
let melgan_model = load_model_file(melgan_filename);
|
||||
|
||||
Self {
|
||||
tacotron_model,
|
||||
melgan_model,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run_tts(&self, text: &str) -> Vec<i16> {
|
||||
println!("Text : {:?}", text);
|
||||
let copied = text.to_string().to_ascii_lowercase();
|
||||
let mut text_buffer : Vec<i64> = Vec::new();
|
||||
|
||||
for ch in copied.chars() {
|
||||
// TODO: HORRIBLE EXPERIMENTAL HACK. Write a formal module to clean and process text
|
||||
let mut v = ch as i64 - 59;
|
||||
if v < 1 {
|
||||
v = 11; // NB: Space
|
||||
}
|
||||
text_buffer.push(v);
|
||||
}
|
||||
|
||||
println!("Text buffer: {:?}", text_buffer);
|
||||
|
||||
let text_tensor = Tensor::of_slice(text_buffer.as_slice());
|
||||
|
||||
println!("Text tensor: {:?}", text_tensor);
|
||||
|
||||
let text_tensor = text_tensor.unsqueeze(0);
|
||||
|
||||
println!("Text tensor unsq: {:?}", text_tensor);
|
||||
|
||||
let mut mel_tensor = self.tacotron_model.forward(&text_tensor);
|
||||
|
||||
//println!("Loading mel...");
|
||||
//let mel_tensor = load_wrapped_tensor_file("/home/bt/dev/tacotron-melgan/saved_mel.pt.containerized.pt");
|
||||
|
||||
println!("\n\n>>> Mel tensor:\n{:?}\n\n", mel_tensor);
|
||||
|
||||
//let repr = mel_tensor.to_string(20000).unwrap();
|
||||
//println!("\n{:?}\n\n", repr);
|
||||
|
||||
println!("Running melgan...");
|
||||
|
||||
let audio_tensor = self.melgan_model.forward(&mel_tensor);
|
||||
|
||||
println!("\n\n>>> Audio tensor:\n{:?}\n\n", audio_tensor);
|
||||
|
||||
audio_tensor_to_audio_signal(audio_tensor)
|
||||
}
|
||||
|
||||
pub fn run_tts_audio(&self, text: &str) -> Vec<u8> {
|
||||
let audio_signal = self.run_tts(text);
|
||||
|
||||
audio_signal_to_wav_bytes(audio_signal)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn audio_signal_to_wav_bytes(audio_signal: Vec<i16>) -> Vec<u8> {
|
||||
let spec = WavSpec {
|
||||
channels: 1,
|
||||
sample_rate: 20000,
|
||||
bits_per_sample: 16,
|
||||
sample_format: SampleFormat::Int,
|
||||
};
|
||||
let bytes: Vec<u8> = Vec::new();
|
||||
let seek: Cursor<Vec<u8>> = Cursor::new(bytes);
|
||||
let mut buffer = BufWriter::new(seek);
|
||||
{
|
||||
let mut writer = WavWriter::new(&mut buffer, spec).unwrap();
|
||||
for s in audio_signal {
|
||||
//let s = s * 0.00001f32; // TODO: Find a more appropriate multiplier
|
||||
writer.write_sample(s).unwrap();
|
||||
}
|
||||
writer.finalize().unwrap(); // TODO: Error
|
||||
}
|
||||
match buffer.into_inner() {
|
||||
Err(_) => { Vec::new() }, // TODO: Error
|
||||
Ok(r) => { r.get_ref().to_vec() },
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
use rocket::{Request, State, Response};
|
||||
use rocket::response::content::Content;
|
||||
use rocket::http::{RawStr, ContentType};
|
||||
use rocket_contrib::json::{Json, JsonValue};
|
||||
use model::{TacoMelModel, load_wrapped_tensor_file, load_model_file, audio_signal_to_wav_bytes};
|
||||
use tch::nn::Module;
|
||||
use melgan::audio_tensor_to_audio_signal;
|
||||
use tch::Tensor;
|
||||
use std::panic;
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct Message {
|
||||
contents: String,
|
||||
}
|
||||
|
||||
#[get("/")]
|
||||
fn get_index() -> String {
|
||||
format!("<b>Hello</b>")
|
||||
}
|
||||
|
||||
#[get("/tts?<text>")]
|
||||
fn get_tts(model: State<TacoMelModel>, text: &RawStr) -> Content<Vec<u8>> {
|
||||
|
||||
let result = panic::catch_unwind(|| {
|
||||
let audio_bytes = model.inner().run_tts_audio(text.as_str());
|
||||
audio_bytes
|
||||
});
|
||||
|
||||
match result {
|
||||
Err(_) => {
|
||||
Content(ContentType::Plain, "there was an error".as_bytes().to_vec())
|
||||
},
|
||||
Ok(audio_bytes) => {
|
||||
Content(ContentType::new("audio", "wav"), audio_bytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn print_mel(tensor: &Tensor, i: i64) {
|
||||
let t = tensor.squeeze();
|
||||
print!("M[{}][0:5] =", i);
|
||||
for j in 0..5 {
|
||||
print!(" {:?}", t.get(i).get(j));
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
fn print_audio(tensor: &Tensor, i: i64) {
|
||||
let t = tensor.squeeze();
|
||||
println!("A[{}] = {:?}", i, t.get(i));
|
||||
}
|
||||
|
||||
fn print_audio_signal(signal: &Vec<i16>, i: usize) {
|
||||
println!("W[{}] = {:?}", i, signal.get(i).unwrap());
|
||||
}
|
||||
|
||||
|
||||
#[get("/tts_test")]
|
||||
fn get_tts_test() -> Content<Vec<u8>> {
|
||||
println!("Loading melgan model...");
|
||||
let melgan_filename = "/home/bt/dev/tacotron-melgan/shared_melgan_container.pt";
|
||||
let melgan_filename = "/home/bt/dev/tacotron-melgan/shared_melgan_container2.pt";
|
||||
let melgan_model = load_model_file(melgan_filename);
|
||||
|
||||
println!("Loading mel...");
|
||||
let mel_tensor = load_wrapped_tensor_file("/home/bt/dev/tacotron-melgan/saved_mel.pt.containerized.pt");
|
||||
|
||||
println!("⚙️");
|
||||
print_mel(&mel_tensor, 0);
|
||||
print_mel(&mel_tensor, 1);
|
||||
print_mel(&mel_tensor, 2);
|
||||
print_mel(&mel_tensor, 3);
|
||||
print_mel(&mel_tensor, 4);
|
||||
print_mel(&mel_tensor, 79);
|
||||
|
||||
//println!("\n\n>>> Mel tensor:\n{:?}\n\n", mel_tensor);
|
||||
|
||||
let audio_tensor = melgan_model.forward(&mel_tensor);
|
||||
|
||||
print_audio(&audio_tensor, 0);
|
||||
print_audio(&audio_tensor, 1);
|
||||
print_audio(&audio_tensor, 2);
|
||||
print_audio(&audio_tensor, 100);
|
||||
print_audio(&audio_tensor, 101);
|
||||
print_audio(&audio_tensor, 102);
|
||||
|
||||
//println!("\n\n>>> Audio tensor:\n{:?}\n\n", mel_tensor);
|
||||
|
||||
let audio_signal = audio_tensor_to_audio_signal(audio_tensor);
|
||||
|
||||
print_audio_signal(&audio_signal, 0);
|
||||
print_audio_signal(&audio_signal, 1);
|
||||
print_audio_signal(&audio_signal, 2);
|
||||
print_audio_signal(&audio_signal, 100);
|
||||
print_audio_signal(&audio_signal, 101);
|
||||
print_audio_signal(&audio_signal, 102);
|
||||
|
||||
let audio_bytes = audio_signal_to_wav_bytes(audio_signal);
|
||||
Content(ContentType::new("audio", "wav"), audio_bytes)
|
||||
}
|
||||
|
||||
#[catch(404)]
|
||||
fn not_found(req: &Request) -> String {
|
||||
format!("I couldn't find '{}'. Try something else?", req.uri())
|
||||
}
|
||||
|
||||
pub fn run_server() {
|
||||
/*let options = Options::Index | Options::DotFiles;
|
||||
rocket::ignite()
|
||||
.mount("/static", StaticFiles::from("/www/public"))
|
||||
.mount("/pub", StaticFiles::new("/www/public", options).rank(-1))*/
|
||||
|
||||
//let tacotron = "/home/bt/dev/voder/tacotron_melgan/tacotron_jit_model_voder_c0cac635.pt";
|
||||
let tacotron = "/home/bt/dev/voder/tacotron_melgan/tacotron_container4.pt";
|
||||
//let melgan = "/home/bt/dev/voder/tacotron_melgan/melgan_jit_model_voder_c0cac635.pt";
|
||||
let melgan = "/home/bt/dev/voder/tacotron_melgan/melgan_container2.pt";
|
||||
let melgan = "/home/bt/dev/tacotron-melgan/shared_melgan_container.pt";
|
||||
let melgan = "/home/bt/dev/tacotron-melgan/shared_melgan_container2.pt";
|
||||
let model = TacoMelModel::create(
|
||||
tacotron,
|
||||
melgan);
|
||||
|
||||
rocket::ignite()
|
||||
.mount("/", routes![
|
||||
get_index,
|
||||
get_tts,
|
||||
get_tts_test,
|
||||
])
|
||||
.manage(model)
|
||||
.register(catchers![not_found])
|
||||
.launch();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
use tch::Tensor;
|
||||
use tch::CModule;
|
||||
use tch;
|
||||
use tch::nn::Module;
|
||||
|
||||
const WRAPPED_MODEL_PATH : &'static str = "/home/bt/dev/voder/tacotron_melgan/tacotron_jit_model_voder_c0cac635.pt";
|
||||
//const EXAMPLE_TEXT_SEQUENCE : &'static str = "/home/bt/dev/voder/data/text/tacotron_text_sequence.pt";
|
||||
const EXAMPLE_TEXT_SEQUENCE : &'static str = "/home/bt/dev/voder/data/text/tacotron_text_sequence.pt.containerized.pt";
|
||||
|
||||
pub fn load_tacotron_model(filename: &str) -> CModule {
|
||||
// TODO
|
||||
println!("Loading model: {}", filename);
|
||||
tch::CModule::load(filename).unwrap()
|
||||
}
|
||||
|
||||
pub fn load_wrapped_text(filename: &str) -> Tensor {
|
||||
println!("Loading wrapped text sequence file: {}", filename);
|
||||
let module = tch::CModule::load(filename).unwrap();
|
||||
let mut temp = Tensor::zeros(
|
||||
&[10, 10, 10],
|
||||
(tch::Kind::Float, tch::Device::Cpu)
|
||||
);
|
||||
module.forward(&temp)
|
||||
}
|
||||
|
||||
|
||||
pub fn run_tacotron() {
|
||||
let mut text_sequence = load_wrapped_text(EXAMPLE_TEXT_SEQUENCE);
|
||||
println!("Got text_sequence: {:?} of dim {}", text_sequence, text_sequence.dim());
|
||||
|
||||
let mut vs = tch::nn::VarStore::new(tch::Device::Cpu);
|
||||
let tacotron_model = load_tacotron_model(WRAPPED_MODEL_PATH);
|
||||
|
||||
for i in 0..10 {
|
||||
println!("Evaluating model... {}", i);
|
||||
let output = tacotron_model.forward(&text_sequence);
|
||||
|
||||
println!("Result tensor: {:?}", output);
|
||||
}
|
||||
}
|
||||
|
||||
/*println!("Tacotron2 + MelGan");
|
||||
|
||||
// Create the model and load the weights from the file.
|
||||
let mut vs = tch::nn::VarStore::new(tch::Device::Cpu);
|
||||
|
||||
println!("Loading Wrapped model: {}", WRAPPED_MODEL_PATH);
|
||||
|
||||
println!("loaded");
|
||||
|
||||
/* Just gotta do this...
|
||||
model = Generator(hp.audio.n_mel_channels).cuda()
|
||||
model.load_state_dict(checkpoint['model_g'])
|
||||
model.eval(inference=False)
|
||||
|
||||
with torch.no_grad():
|
||||
for melpath in tqdm.tqdm(glob.glob(os.path.join(args.input_folder, '*.mel'))):
|
||||
mel = torch.load(melpath)
|
||||
if len(mel.shape) == 2:
|
||||
mel = mel.unsqueeze(0)
|
||||
|
||||
filename = melpath.replace('.mel', '_reconstructed_epoch%04d.png' % checkpoint['epoch'])
|
||||
#render_histogram(mel, filename)
|
||||
mel = mel.cuda()
|
||||
|
||||
audio = model.inference(mel)
|
||||
audio = audio.cpu().detach().numpy()
|
||||
|
||||
out_path = melpath.replace('.mel', '_reconstructed_epoch%04d.wav' % checkpoint['epoch'])
|
||||
write(out_path, hp.audio.sampling_rate, audio)
|
||||
*/
|
||||
|
||||
//vs.load(WRAPPED_MODEL_PATH).unwrap();
|
||||
|
||||
println!("Loading trained model...");
|
||||
let model = tch::CModule::load(WRAPPED_MODEL_PATH).unwrap();
|
||||
|
||||
// NB: This is just random data for now. Kind of mel shaped since it's an image.
|
||||
//let path = "/home/bt/Downloads/virtual-studio-design.jpg";
|
||||
//let image = imagenet::load_image_and_resize(path, 100, 100).unwrap();
|
||||
//let image = imagenet::load_image(EXAMPLE_MEL_PATH).unwrap();
|
||||
|
||||
println!("Loading mel file...");
|
||||
let mel_file = Tensor::load(EXAMPLE_MEL_PATH).unwrap();
|
||||
|
||||
println!("Forwarding mel to model...");
|
||||
let output = model.forward_ts(&[mel_file.unsqueeze(0)]).unwrap();
|
||||
|
||||
|
||||
/*// TODO: Now how do I evaluate the model?
|
||||
//let resnet18 = tch::vision::resnet::resnet18(&vs.root(), imagenet::CLASS_COUNT);
|
||||
|
||||
let path = "/home/bt/Downloads/virtual-studio-design.jpg";
|
||||
let image = imagenet::load_image_and_resize(path, 100, 100).unwrap();
|
||||
|
||||
// NB: This works
|
||||
let net = Box::new(resnet::resnet18(&vs.root(), imagenet::CLASS_COUNT));
|
||||
|
||||
for _ in 0..10 {
|
||||
let output = net
|
||||
.forward_t(&image.unsqueeze(0), /*train=*/ false)
|
||||
.softmax(-1, tch::Kind::Float); // Convert to probability.
|
||||
|
||||
for (probability, class) in imagenet::top(&output, 5).iter() {
|
||||
println!("{:50} {:5.2}%", class, 100.0 * probability)
|
||||
}
|
||||
}*/
|
||||
|
||||
/*// Apply the forward pass of the model to get the logits.
|
||||
let output = net
|
||||
.forward_t(&image.unsqueeze(0), /*train=*/ false)
|
||||
.softmax(-1, tch::Kind::Float); // Convert to probability.
|
||||
|
||||
// Print the top 5 categories for this image.
|
||||
Ok(())*/
|
||||
*/
|
||||
@@ -1,604 +0,0 @@
|
||||
# Adapted from nvidia's tacotron2 codebase
|
||||
|
||||
from math import sqrt
|
||||
import torch
|
||||
from torch.autograd import Variable
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
from layers import ConvNorm, LinearNorm
|
||||
#from utils import to_gpu, get_mask_from_lengths
|
||||
|
||||
class LocationLayer(nn.Module):
|
||||
def __init__(self, attention_n_filters, attention_kernel_size,
|
||||
attention_dim):
|
||||
super(LocationLayer, self).__init__()
|
||||
padding = int((attention_kernel_size - 1) / 2)
|
||||
self.location_conv = ConvNorm(2, attention_n_filters,
|
||||
kernel_size=attention_kernel_size,
|
||||
padding=padding, bias=False, stride=1,
|
||||
dilation=1)
|
||||
self.location_dense = LinearNorm(attention_n_filters, attention_dim,
|
||||
bias=False, w_init_gain='tanh')
|
||||
|
||||
def forward(self, attention_weights_cat):
|
||||
processed_attention = self.location_conv(attention_weights_cat)
|
||||
processed_attention = processed_attention.transpose(1, 2)
|
||||
processed_attention = self.location_dense(processed_attention)
|
||||
return processed_attention
|
||||
|
||||
|
||||
class Attention(nn.Module):
|
||||
def __init__(self, attention_rnn_dim, embedding_dim, attention_dim,
|
||||
attention_location_n_filters, attention_location_kernel_size):
|
||||
super(Attention, self).__init__()
|
||||
self.query_layer = LinearNorm(attention_rnn_dim, attention_dim,
|
||||
bias=False, w_init_gain='tanh')
|
||||
self.memory_layer = LinearNorm(embedding_dim, attention_dim, bias=False,
|
||||
w_init_gain='tanh')
|
||||
self.v = LinearNorm(attention_dim, 1, bias=False)
|
||||
self.location_layer = LocationLayer(attention_location_n_filters,
|
||||
attention_location_kernel_size,
|
||||
attention_dim)
|
||||
self.score_mask_value = -float("inf")
|
||||
|
||||
def get_alignment_energies(self, query, processed_memory,
|
||||
attention_weights_cat):
|
||||
"""
|
||||
PARAMS
|
||||
------
|
||||
query: decoder output (batch, n_mel_channels * n_frames_per_step)
|
||||
processed_memory: processed encoder outputs (B, T_in, attention_dim)
|
||||
attention_weights_cat: cumulative and prev. att weights (B, 2, max_time)
|
||||
|
||||
RETURNS
|
||||
-------
|
||||
alignment (batch, max_time)
|
||||
"""
|
||||
|
||||
processed_query = self.query_layer(query.unsqueeze(1))
|
||||
processed_attention_weights = self.location_layer(attention_weights_cat)
|
||||
energies = self.v(torch.tanh(
|
||||
processed_query + processed_attention_weights + processed_memory))
|
||||
|
||||
energies = energies.squeeze(-1)
|
||||
return energies
|
||||
|
||||
def forward(self, attention_hidden_state, memory, processed_memory,
|
||||
#attention_weights_cat, mask):
|
||||
attention_weights_cat):
|
||||
"""
|
||||
PARAMS
|
||||
------
|
||||
attention_hidden_state: attention rnn last output
|
||||
memory: encoder outputs
|
||||
processed_memory: processed encoder outputs
|
||||
attention_weights_cat: previous and cummulative attention weights
|
||||
mask: binary mask for padded data
|
||||
"""
|
||||
alignment = self.get_alignment_energies(
|
||||
attention_hidden_state, processed_memory, attention_weights_cat)
|
||||
|
||||
#if mask is not None:
|
||||
# alignment.data.masked_fill_(mask, self.score_mask_value)
|
||||
|
||||
attention_weights = F.softmax(alignment, dim=1)
|
||||
attention_context = torch.bmm(attention_weights.unsqueeze(1), memory)
|
||||
attention_context = attention_context.squeeze(1)
|
||||
|
||||
return attention_context, attention_weights
|
||||
|
||||
|
||||
class Prenet(nn.Module):
|
||||
def __init__(self, in_dim, sizes):
|
||||
super(Prenet, self).__init__()
|
||||
in_sizes = [in_dim] + sizes[:-1]
|
||||
self.layers = nn.ModuleList(
|
||||
[LinearNorm(in_size, out_size, bias=False)
|
||||
for (in_size, out_size) in zip(in_sizes, sizes)])
|
||||
|
||||
def forward(self, x):
|
||||
for linear in self.layers:
|
||||
x = F.dropout(F.relu(linear(x)), p=0.5, training=True)
|
||||
return x
|
||||
|
||||
|
||||
class Postnet(nn.Module):
|
||||
"""Postnet
|
||||
- Five 1-d convolution with 512 channels and kernel size 5
|
||||
"""
|
||||
|
||||
def __init__(self, hparams):
|
||||
super(Postnet, self).__init__()
|
||||
self.convolutions = nn.ModuleList()
|
||||
|
||||
# TODO(TEMP)
|
||||
"""
|
||||
self.convolutions.append(
|
||||
nn.Sequential(
|
||||
ConvNorm(hparams.n_mel_channels, hparams.postnet_embedding_dim,
|
||||
kernel_size=hparams.postnet_kernel_size, stride=1,
|
||||
padding=int((hparams.postnet_kernel_size - 1) / 2),
|
||||
dilation=1, w_init_gain='tanh'),
|
||||
nn.BatchNorm1d(hparams.postnet_embedding_dim))
|
||||
)
|
||||
|
||||
for i in range(1, hparams.postnet_n_convolutions - 1):
|
||||
self.convolutions.append(
|
||||
nn.Sequential(
|
||||
ConvNorm(hparams.postnet_embedding_dim,
|
||||
hparams.postnet_embedding_dim,
|
||||
kernel_size=hparams.postnet_kernel_size, stride=1,
|
||||
padding=int((hparams.postnet_kernel_size - 1) / 2),
|
||||
dilation=1, w_init_gain='tanh'),
|
||||
nn.BatchNorm1d(hparams.postnet_embedding_dim))
|
||||
)
|
||||
|
||||
self.convolutions.append(
|
||||
nn.Sequential(
|
||||
ConvNorm(hparams.postnet_embedding_dim, hparams.n_mel_channels,
|
||||
kernel_size=hparams.postnet_kernel_size, stride=1,
|
||||
padding=int((hparams.postnet_kernel_size - 1) / 2),
|
||||
dilation=1, w_init_gain='linear'),
|
||||
nn.BatchNorm1d(hparams.n_mel_channels))
|
||||
)
|
||||
"""
|
||||
|
||||
def forward(self, x):
|
||||
for i in range(len(self.convolutions) - 1):
|
||||
x = F.dropout(torch.tanh(self.convolutions[i](x)), 0.5, self.training)
|
||||
x = F.dropout(self.convolutions[-1](x), 0.5, self.training)
|
||||
|
||||
return x
|
||||
|
||||
|
||||
class Encoder(nn.Module):
|
||||
"""Encoder module:
|
||||
- Three 1-d convolution banks
|
||||
- Bidirectional LSTM
|
||||
"""
|
||||
def __init__(self, hparams):
|
||||
super(Encoder, self).__init__()
|
||||
|
||||
convolutions = []
|
||||
for _ in range(hparams.encoder_n_convolutions):
|
||||
conv_layer = nn.Sequential(
|
||||
ConvNorm(hparams.encoder_embedding_dim,
|
||||
hparams.encoder_embedding_dim,
|
||||
kernel_size=hparams.encoder_kernel_size, stride=1,
|
||||
padding=int((hparams.encoder_kernel_size - 1) / 2),
|
||||
dilation=1, w_init_gain='relu'),
|
||||
nn.BatchNorm1d(hparams.encoder_embedding_dim))
|
||||
convolutions.append(conv_layer)
|
||||
self.convolutions = nn.ModuleList(convolutions)
|
||||
|
||||
self.lstm = nn.LSTM(hparams.encoder_embedding_dim,
|
||||
int(hparams.encoder_embedding_dim / 2), 1,
|
||||
batch_first=True, bidirectional=True)
|
||||
|
||||
# def forward(self, x, input_lengths):
|
||||
# for conv in self.convolutions:
|
||||
# x = F.dropout(F.relu(conv(x)), 0.5, self.training)
|
||||
|
||||
# x = x.transpose(1, 2)
|
||||
|
||||
# # pytorch tensor are not reversible, hence the conversion
|
||||
# #input_lengths = input_lengths.cpu().numpy()
|
||||
# x = nn.utils.rnn.pack_padded_sequence(
|
||||
# x, input_lengths, batch_first=True)
|
||||
|
||||
# self.lstm.flatten_parameters()
|
||||
# outputs, _ = self.lstm(x)
|
||||
|
||||
# outputs, _ = nn.utils.rnn.pad_packed_sequence(
|
||||
# outputs, batch_first=True)
|
||||
|
||||
# return outputs
|
||||
|
||||
# NB: This used to be 'inference'
|
||||
def forward(self, x):
|
||||
for conv in self.convolutions:
|
||||
x = F.dropout(F.relu(conv(x)), 0.5, self.training)
|
||||
|
||||
x = x.transpose(1, 2)
|
||||
|
||||
# NB(bt): Don't need to flatten, per the method comments:
|
||||
# Right now, this works only if the module is on the GPU and cuDNN is enabled.
|
||||
#self.lstm.flatten_parameters()
|
||||
outputs, _ = self.lstm(x)
|
||||
|
||||
return outputs
|
||||
|
||||
class Decoder(nn.Module):
|
||||
def __init__(self, hparams):
|
||||
super(Decoder, self).__init__()
|
||||
|
||||
# NB(bt): Initializing these upfront because JIT script fails otherwise:
|
||||
# "Tried to set nonexistent attribute: attention_hidden.
|
||||
# Did you forget to initialize it in __init__()?:"
|
||||
self.attention_hidden = torch.zeros(1,1)
|
||||
self.attention_cell = torch.zeros(1, 1)
|
||||
self.decoder_hidden = torch.zeros(1, 1)
|
||||
self.decoder_cell = torch.zeros(1, 1)
|
||||
self.attention_weights = torch.zeros(1, 1)
|
||||
self.attention_weights_cum = torch.zeros(1, 1)
|
||||
self.attention_context = torch.zeros(1, 1)
|
||||
self.memory = torch.zeros(1, 1)
|
||||
self.processed_memory = torch.zeros(1, 1)
|
||||
|
||||
self.n_mel_channels = hparams.n_mel_channels
|
||||
self.n_frames_per_step = hparams.n_frames_per_step
|
||||
self.encoder_embedding_dim = hparams.encoder_embedding_dim
|
||||
self.attention_rnn_dim = hparams.attention_rnn_dim
|
||||
self.decoder_rnn_dim = hparams.decoder_rnn_dim
|
||||
self.prenet_dim = hparams.prenet_dim
|
||||
self.max_decoder_steps = hparams.max_decoder_steps
|
||||
self.gate_threshold = hparams.gate_threshold
|
||||
self.p_attention_dropout = hparams.p_attention_dropout
|
||||
self.p_decoder_dropout = hparams.p_decoder_dropout
|
||||
|
||||
self.prenet = Prenet(
|
||||
hparams.n_mel_channels * hparams.n_frames_per_step,
|
||||
[hparams.prenet_dim, hparams.prenet_dim])
|
||||
|
||||
self.attention_rnn = nn.LSTMCell(
|
||||
hparams.prenet_dim + hparams.encoder_embedding_dim,
|
||||
hparams.attention_rnn_dim)
|
||||
|
||||
self.attention_layer = Attention(
|
||||
hparams.attention_rnn_dim, hparams.encoder_embedding_dim,
|
||||
hparams.attention_dim, hparams.attention_location_n_filters,
|
||||
hparams.attention_location_kernel_size)
|
||||
|
||||
self.decoder_rnn = nn.LSTMCell(
|
||||
hparams.attention_rnn_dim + hparams.encoder_embedding_dim,
|
||||
hparams.decoder_rnn_dim, 1)
|
||||
|
||||
self.linear_projection = LinearNorm(
|
||||
hparams.decoder_rnn_dim + hparams.encoder_embedding_dim,
|
||||
hparams.n_mel_channels * hparams.n_frames_per_step)
|
||||
|
||||
self.gate_layer = LinearNorm(
|
||||
hparams.decoder_rnn_dim + hparams.encoder_embedding_dim, 1,
|
||||
bias=True, w_init_gain='sigmoid')
|
||||
|
||||
def get_go_frame(self, memory):
|
||||
""" Gets all zeros frames to use as first decoder input
|
||||
PARAMS
|
||||
------
|
||||
memory: decoder outputs
|
||||
|
||||
RETURNS
|
||||
-------
|
||||
decoder_input: all zeros frames
|
||||
"""
|
||||
|
||||
B = memory.size(0)
|
||||
# NB(bt): Rewriting this as it doesn't work with scripting (but
|
||||
# does with tracing).
|
||||
# mem = memory.data.new(B, self.n_mel_channels * self.n_frames_per_step)
|
||||
# decoder_input = Variable(mem.zero_())
|
||||
mem = torch.zeros(B, self.n_mel_channels * self.n_frames_per_step)
|
||||
# NB(bt): I don't think I need a variable as we don't need to do backprop.
|
||||
# decoder_input = Variable(mem)
|
||||
# return decoder_input
|
||||
return mem
|
||||
|
||||
#def initialize_decoder_states(self, memory, mask):
|
||||
def initialize_decoder_states(self, memory):
|
||||
""" Initializes attention rnn states, decoder rnn states, attention
|
||||
weights, attention cumulative weights, attention context, stores memory
|
||||
and stores processed memory
|
||||
PARAMS
|
||||
------
|
||||
memory: Encoder outputs
|
||||
mask: Mask for padded data if training, expects None for inference
|
||||
"""
|
||||
B = memory.size(0)
|
||||
MAX_TIME = memory.size(1)
|
||||
|
||||
# NB(bt): Rewriting for scripting
|
||||
#self.attention_hidden = Variable(memory.data.new(
|
||||
# B, self.attention_rnn_dim).zero_())
|
||||
self.attention_hidden = torch.zeros(B, self.attention_rnn_dim)
|
||||
#self.attention_cell = Variable(memory.data.new(
|
||||
# B, self.attention_rnn_dim).zero_())
|
||||
self.attention_cell = torch.zeros(B, self.attention_rnn_dim)
|
||||
|
||||
#self.decoder_hidden = Variable(memory.data.new(
|
||||
# B, self.decoder_rnn_dim).zero_())
|
||||
self.decoder_hidden = torch.zeros(B, self.decoder_rnn_dim)
|
||||
#self.decoder_cell = Variable(memory.data.new(
|
||||
# B, self.decoder_rnn_dim).zero_())
|
||||
self.decoder_cell = torch.zeros(B, self.decoder_rnn_dim)
|
||||
|
||||
#self.attention_weights = Variable(memory.data.new(
|
||||
# B, MAX_TIME).zero_())
|
||||
self.attention_weights = torch.zeros(B, MAX_TIME)
|
||||
#self.attention_weights_cum = Variable(memory.data.new(
|
||||
# B, MAX_TIME).zero_())
|
||||
self.attention_weights_cum = torch.zeros(B, MAX_TIME)
|
||||
#self.attention_context = Variable(memory.data.new(
|
||||
# B, self.encoder_embedding_dim).zero_())
|
||||
self.attention_context = torch.zeros(B, self.encoder_embedding_dim)
|
||||
|
||||
self.memory = memory
|
||||
self.processed_memory = self.attention_layer.memory_layer(memory)
|
||||
# NB(bt): Can't set to None in the CTOR as it gets optimized out.
|
||||
# self.mask = mask
|
||||
|
||||
def parse_decoder_inputs(self, decoder_inputs):
|
||||
""" Prepares decoder inputs, i.e. mel outputs
|
||||
PARAMS
|
||||
------
|
||||
decoder_inputs: inputs used for teacher-forced training, i.e. mel-specs
|
||||
|
||||
RETURNS
|
||||
-------
|
||||
inputs: processed decoder inputs
|
||||
|
||||
"""
|
||||
# (B, n_mel_channels, T_out) -> (B, T_out, n_mel_channels)
|
||||
decoder_inputs = decoder_inputs.transpose(1, 2)
|
||||
decoder_inputs = decoder_inputs.view(
|
||||
decoder_inputs.size(0),
|
||||
int(decoder_inputs.size(1)/self.n_frames_per_step), -1)
|
||||
# (B, T_out, n_mel_channels) -> (T_out, B, n_mel_channels)
|
||||
decoder_inputs = decoder_inputs.transpose(0, 1)
|
||||
return decoder_inputs
|
||||
|
||||
def parse_decoder_outputs(self, mel_outputs, gate_outputs, alignments):
|
||||
""" Prepares decoder outputs for output
|
||||
PARAMS
|
||||
------
|
||||
mel_outputs:
|
||||
gate_outputs: gate output energies
|
||||
alignments:
|
||||
|
||||
RETURNS
|
||||
-------
|
||||
mel_outputs:
|
||||
gate_outpust: gate output energies
|
||||
alignments:
|
||||
"""
|
||||
# type: (List[torch.Tensor], List[torch.Tensor], List[torch.Tensor]) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
|
||||
# (T_out, B) -> (B, T_out)
|
||||
alignments = torch.stack(alignments).transpose(0, 1)
|
||||
# (T_out, B) -> (B, T_out)
|
||||
gate_outputs = torch.stack(gate_outputs).transpose(0, 1)
|
||||
gate_outputs = gate_outputs.contiguous()
|
||||
# (T_out, B, n_mel_channels) -> (B, T_out, n_mel_channels)
|
||||
mel_outputs = torch.stack(mel_outputs).transpose(0, 1).contiguous()
|
||||
# decouple frames per step
|
||||
mel_outputs = mel_outputs.view(
|
||||
mel_outputs.size(0), -1, self.n_mel_channels)
|
||||
# (B, T_out, n_mel_channels) -> (B, n_mel_channels, T_out)
|
||||
mel_outputs = mel_outputs.transpose(1, 2)
|
||||
|
||||
return mel_outputs, gate_outputs, alignments
|
||||
|
||||
def decode(self, decoder_input):
|
||||
""" Decoder step using stored states, attention and memory
|
||||
PARAMS
|
||||
------
|
||||
decoder_input: previous mel output
|
||||
|
||||
RETURNS
|
||||
-------
|
||||
mel_output:
|
||||
gate_output: gate output energies
|
||||
attention_weights:
|
||||
"""
|
||||
cell_input = torch.cat((decoder_input, self.attention_context), -1)
|
||||
|
||||
#self.attention_hidden, self.attention_cell = self.attention_rnn(
|
||||
results = self.attention_rnn(cell_input, (self.attention_hidden, self.attention_cell))
|
||||
self.attention_hidden = results[0]
|
||||
self.attention_cell = results[1]
|
||||
self.attention_hidden = F.dropout(
|
||||
self.attention_hidden, self.p_attention_dropout, self.training)
|
||||
|
||||
attention_weights_cat = torch.cat(
|
||||
(self.attention_weights.unsqueeze(1),
|
||||
self.attention_weights_cum.unsqueeze(1)), dim=1)
|
||||
#self.attention_context, self.attention_weights = self.attention_layer(
|
||||
results = self.attention_layer(
|
||||
self.attention_hidden, self.memory, self.processed_memory,
|
||||
#attention_weights_cat, self.mask)
|
||||
attention_weights_cat)
|
||||
self.attention_context = results[0]
|
||||
self.attention_weights = results[1]
|
||||
|
||||
self.attention_weights_cum += self.attention_weights
|
||||
decoder_input = torch.cat(
|
||||
(self.attention_hidden, self.attention_context), -1)
|
||||
#self.decoder_hidden, self.decoder_cell = self.decoder_rnn(
|
||||
results = self.decoder_rnn(
|
||||
decoder_input, (self.decoder_hidden, self.decoder_cell))
|
||||
self.decoder_hidden = results[0]
|
||||
self.decoder_cell = results[1]
|
||||
self.decoder_hidden = F.dropout(
|
||||
self.decoder_hidden, self.p_decoder_dropout, self.training)
|
||||
|
||||
decoder_hidden_attention_context = torch.cat(
|
||||
(self.decoder_hidden, self.attention_context), dim=1)
|
||||
decoder_output = self.linear_projection(
|
||||
decoder_hidden_attention_context)
|
||||
|
||||
gate_prediction = self.gate_layer(decoder_hidden_attention_context)
|
||||
return decoder_output, gate_prediction, self.attention_weights
|
||||
|
||||
# def forward(self, memory, decoder_inputs, memory_lengths):
|
||||
# """ Decoder forward pass for training
|
||||
# PARAMS
|
||||
# ------
|
||||
# memory: Encoder outputs
|
||||
# decoder_inputs: Decoder inputs for teacher forcing. i.e. mel-specs
|
||||
# memory_lengths: Encoder output lengths for attention masking.
|
||||
#
|
||||
# RETURNS
|
||||
# -------
|
||||
# mel_outputs: mel outputs from the decoder
|
||||
# gate_outputs: gate outputs from the decoder
|
||||
# alignments: sequence of attention weights from the decoder
|
||||
# """
|
||||
#
|
||||
# decoder_input = self.get_go_frame(memory).unsqueeze(0)
|
||||
# decoder_inputs = self.parse_decoder_inputs(decoder_inputs)
|
||||
# decoder_inputs = torch.cat((decoder_input, decoder_inputs), dim=0)
|
||||
# decoder_inputs = self.prenet(decoder_inputs)
|
||||
#
|
||||
# self.initialize_decoder_states(
|
||||
# memory, mask=~get_mask_from_lengths(memory_lengths))
|
||||
#
|
||||
# mel_outputs, gate_outputs, alignments = [], [], []
|
||||
# while len(mel_outputs) < decoder_inputs.size(0) - 1:
|
||||
# decoder_input = decoder_inputs[len(mel_outputs)]
|
||||
# mel_output, gate_output, attention_weights = self.decode(
|
||||
# decoder_input)
|
||||
# mel_outputs += [mel_output.squeeze(1)]
|
||||
# gate_outputs += [gate_output.squeeze()]
|
||||
# alignments += [attention_weights]
|
||||
#
|
||||
# mel_outputs, gate_outputs, alignments = self.parse_decoder_outputs(
|
||||
# mel_outputs, gate_outputs, alignments)
|
||||
#
|
||||
# return mel_outputs, gate_outputs, alignments
|
||||
|
||||
def forward(self, memory):
|
||||
""" Decoder inference
|
||||
PARAMS
|
||||
------
|
||||
memory: Encoder outputs
|
||||
|
||||
RETURNS
|
||||
-------
|
||||
mel_outputs: mel outputs from the decoder
|
||||
gate_outputs: gate outputs from the decoder
|
||||
alignments: sequence of attention weights from the decoder
|
||||
"""
|
||||
decoder_input = self.get_go_frame(memory)
|
||||
|
||||
#self.initialize_decoder_states(memory, mask=None)
|
||||
self.initialize_decoder_states(memory)
|
||||
|
||||
mel_outputs, gate_outputs, alignments = [], [], []
|
||||
while True:
|
||||
decoder_input = self.prenet(decoder_input)
|
||||
mel_output, gate_output, alignment = self.decode(decoder_input)
|
||||
|
||||
mel_outputs += [mel_output.squeeze(1)]
|
||||
gate_outputs += [gate_output]
|
||||
alignments += [alignment]
|
||||
|
||||
if torch.sigmoid(gate_output.data) > self.gate_threshold:
|
||||
break
|
||||
elif len(mel_outputs) == self.max_decoder_steps:
|
||||
print("Warning! Reached max decoder steps")
|
||||
break
|
||||
|
||||
decoder_input = mel_output
|
||||
|
||||
mel_outputs, gate_outputs, alignments = self.parse_decoder_outputs(
|
||||
mel_outputs, gate_outputs, alignments)
|
||||
|
||||
return mel_outputs, gate_outputs, alignments
|
||||
|
||||
class Tacotron2(nn.Module):
|
||||
def __init__(self, hparams):
|
||||
super(Tacotron2, self).__init__()
|
||||
self.mask_padding = hparams.mask_padding
|
||||
self.fp16_run = hparams.fp16_run
|
||||
self.n_mel_channels = hparams.n_mel_channels
|
||||
self.n_frames_per_step = hparams.n_frames_per_step
|
||||
|
||||
# https://pytorch.org/docs/stable/tensors.html
|
||||
#torch.set_default_tensor_type('torch.HalfTensor')
|
||||
#torch.set_default_tensor_type('torch.DoubleTensor')
|
||||
|
||||
self.embedding = nn.Embedding(
|
||||
hparams.n_symbols, hparams.symbols_embedding_dim)
|
||||
|
||||
std = sqrt(2.0 / (hparams.n_symbols + hparams.symbols_embedding_dim))
|
||||
val = sqrt(3.0) * std # uniform bounds for std
|
||||
self.embedding.weight.data.uniform_(-val, val)
|
||||
self.encoder = Encoder(hparams)
|
||||
self.decoder = Decoder(hparams)
|
||||
# NB(bt): Postnet does not seem to have different output per inference experiments
|
||||
#self.postnet = Postnet(hparams)
|
||||
|
||||
# def parse_batch(self, batch):
|
||||
# text_padded, input_lengths, mel_padded, gate_padded, \
|
||||
# output_lengths = batch
|
||||
# text_padded = to_gpu(text_padded).long()
|
||||
# input_lengths = to_gpu(input_lengths).long()
|
||||
# max_len = torch.max(input_lengths.data).item()
|
||||
# mel_padded = to_gpu(mel_padded).float()
|
||||
# gate_padded = to_gpu(gate_padded).float()
|
||||
# output_lengths = to_gpu(output_lengths).long()
|
||||
#
|
||||
# return (
|
||||
# (text_padded, input_lengths, mel_padded, max_len, output_lengths),
|
||||
# (mel_padded, gate_padded))
|
||||
#
|
||||
#def parse_output(self, outputs, output_lengths=None):
|
||||
def parse_output(self, outputs):
|
||||
# type: (List[torch.Tensor]) -> List[torch.Tensor]
|
||||
# NB(bt): Not calling this currently
|
||||
"""
|
||||
if self.mask_padding and output_lengths is not None:
|
||||
mask = ~get_mask_from_lengths(output_lengths)
|
||||
mask = mask.expand(self.n_mel_channels, mask.size(0), mask.size(1))
|
||||
mask = mask.permute(1, 0, 2)
|
||||
|
||||
outputs[0].data.masked_fill_(mask, 0.0)
|
||||
# NB(bt): No postnet. outputs[1].data.masked_fill_(mask, 0.0)
|
||||
#outputs[2].data.masked_fill_(mask[:, 0, :], 1e3) # gate energies
|
||||
outputs[1].data.masked_fill_(mask[:, 0, :], 1e3) # gate energies
|
||||
"""
|
||||
|
||||
return outputs
|
||||
|
||||
"""
|
||||
#def forward(self, inputs):
|
||||
def forward(self, text_inputs, text_lengths, mels, max_len, output_lengths):
|
||||
#text_inputs, text_lengths, mels, max_len, output_lengths = inputs
|
||||
text_lengths, output_lengths = text_lengths.data, output_lengths.data
|
||||
|
||||
embedded_inputs = self.embedding(text_inputs).transpose(1, 2)
|
||||
|
||||
encoder_outputs = self.encoder(embedded_inputs, text_lengths)
|
||||
|
||||
mel_outputs, gate_outputs, alignments = self.decoder(
|
||||
encoder_outputs, mels, memory_lengths=text_lengths)
|
||||
|
||||
mel_outputs_postnet = self.postnet(mel_outputs)
|
||||
mel_outputs_postnet = mel_outputs + mel_outputs_postnet
|
||||
|
||||
return self.parse_output(
|
||||
[mel_outputs, mel_outputs_postnet, gate_outputs, alignments],
|
||||
output_lengths)
|
||||
"""
|
||||
|
||||
# NB: This was the old 'inference' method. It's the single entrypoint into mel inference.
|
||||
def forward(self, inputs):
|
||||
#torch.set_default_tensor_type('torch.DoubleTensor')
|
||||
temp = self.embedding(inputs)
|
||||
embedded_inputs = temp.transpose(1, 2)
|
||||
encoder_outputs = self.encoder.forward(embedded_inputs)
|
||||
mel_outputs, gate_outputs, alignments = self.decoder.forward(encoder_outputs)
|
||||
|
||||
# TODO(bt): The postnet seemed to produce the *same* output, but perhaps not necessary
|
||||
#mel_outputs_postnet = self.postnet(mel_outputs)
|
||||
#mel_outputs_postnet = mel_outputs + mel_outputs_postnet
|
||||
|
||||
outputs = self.parse_output(
|
||||
[
|
||||
mel_outputs,
|
||||
#mel_outputs_postnet,
|
||||
gate_outputs,
|
||||
alignments
|
||||
])
|
||||
|
||||
#return outputs
|
||||
return outputs[0].float()
|
||||
@@ -1,148 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# From: https://github.com/pytorch/pytorch/issues/20356#issuecomment-545572400
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from collections import OrderedDict
|
||||
import re
|
||||
import yaml
|
||||
|
||||
#torch.set_default_tensor_type('torch.DoubleTensor')
|
||||
|
||||
class ResStack(nn.Module):
|
||||
def __init__(self, channel):
|
||||
super(ResStack, self).__init__()
|
||||
|
||||
block_list = [
|
||||
nn.Sequential(
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.ReflectionPad1d(3**i),
|
||||
nn.utils.weight_norm(nn.Conv1d(channel, channel, kernel_size=3, dilation=3**i)),
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.utils.weight_norm(nn.Conv1d(channel, channel, kernel_size=1)),
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
|
||||
self.block0 = block_list[0]
|
||||
self.block1 = block_list[1]
|
||||
self.block2 = block_list[2]
|
||||
|
||||
shortcut_list = [
|
||||
nn.utils.weight_norm(nn.Conv1d(channel, channel, kernel_size=1))
|
||||
for i in range(3)
|
||||
]
|
||||
|
||||
self.shortcut0 = shortcut_list[0]
|
||||
self.shortcut1 = shortcut_list[1]
|
||||
self.shortcut2 = shortcut_list[2]
|
||||
|
||||
|
||||
def forward(self, x):
|
||||
x = self.shortcut0(x) + self.block0(x)
|
||||
x = self.shortcut1(x) + self.block1(x)
|
||||
x = self.shortcut2(x) + self.block2(x)
|
||||
return x
|
||||
|
||||
class Container(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
mel_channel = 80 # TODO: Load from yaml
|
||||
|
||||
self.generator = nn.Sequential(
|
||||
nn.ReflectionPad1d(3),
|
||||
nn.utils.weight_norm(nn.Conv1d(mel_channel, 512, kernel_size=7, stride=1)),
|
||||
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.utils.weight_norm(nn.ConvTranspose1d(512, 256, kernel_size=16, stride=8, padding=4)),
|
||||
|
||||
ResStack(256),
|
||||
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.utils.weight_norm(nn.ConvTranspose1d(256, 128, kernel_size=16, stride=8, padding=4)),
|
||||
|
||||
ResStack(128),
|
||||
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.utils.weight_norm(nn.ConvTranspose1d(128, 64, kernel_size=4, stride=2, padding=1)),
|
||||
|
||||
ResStack(64),
|
||||
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.utils.weight_norm(nn.ConvTranspose1d(64, 32, kernel_size=4, stride=2, padding=1)),
|
||||
|
||||
ResStack(32),
|
||||
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.ReflectionPad1d(3),
|
||||
nn.utils.weight_norm(nn.Conv1d(32, 1, kernel_size=7, stride=1)),
|
||||
nn.Tanh(),
|
||||
)
|
||||
|
||||
# TODO: Looks unnecessary. Remove once this works.
|
||||
def eval(self):
|
||||
super(Container, self).eval()
|
||||
|
||||
|
||||
def forward(self, mel):
|
||||
mel = (mel + 5.0) / 5.0 # roughly normalize spectrogram
|
||||
audio = self.generator(mel)
|
||||
|
||||
# From inference() method:
|
||||
audio = audio.squeeze()
|
||||
#MAX_WAV_VALUE = 32768.0
|
||||
#audio = MAX_WAV_VALUE * audio
|
||||
|
||||
#return audio.float()
|
||||
return audio
|
||||
|
||||
|
||||
print('Loading melgan model...')
|
||||
melgan_model_file = '/home/bt/models/melgan-swpark/firstgo_a7c2351_3650.pt'
|
||||
melgan_model_file = '/home/bt/models/melgan-swpark/ljs-high-pitch_8741385_6975.pt'
|
||||
melgan_model = torch.load(melgan_model_file, map_location=torch.device('cpu'))
|
||||
|
||||
module = Container()
|
||||
|
||||
# NB: This is done so we can rename the model components above since `zip` won't work :(
|
||||
BLOCK_REGEX = re.compile('blocks\.(\d+)\.')
|
||||
SHORTCUT_REGEX = re.compile('shortcuts\.(\d+)\.')
|
||||
|
||||
new_model_g = [] # Rebuild the ordered dict
|
||||
for key, value in melgan_model['model_g'].items():
|
||||
if 'blocks.' in key:
|
||||
new_key = BLOCK_REGEX.sub(r'block\1.', key)
|
||||
key = new_key
|
||||
elif 'shortcuts.' in key:
|
||||
new_key = SHORTCUT_REGEX.sub(r'shortcut\1.', key)
|
||||
key = new_key
|
||||
new_model_g.append((key, value))
|
||||
|
||||
|
||||
melgan_model['model_g'] = OrderedDict(new_model_g)
|
||||
|
||||
print('Load state dict...')
|
||||
module.load_state_dict(melgan_model['model_g'])
|
||||
|
||||
#output_filename = 'melgan_container2.pt'
|
||||
output_filename = 'melgan_ljs-high-pitch_container.pt'
|
||||
|
||||
# NB: Tracing evaluates the model on input and unrolls and hardcodes branching
|
||||
# and loops. Scripting allows these to remain by converting the entire program
|
||||
# to TorchScript. Scripting seems MUCH faster, is also harder get working.
|
||||
trace_model = True
|
||||
if trace_model:
|
||||
print('Tracing model and saving as: {}'.format(output_filename))
|
||||
mel_file = '/home/bt/dev/voder/data/mels/LJ002-0320.mel'
|
||||
mel = torch.load(mel_file, map_location=torch.device('cpu'))
|
||||
# NB: Getting `Tracing failed sanity checks! Graphs differed across invocations!`
|
||||
traced_script_module = torch.jit.trace(module, mel, check_trace=True)
|
||||
traced_script_module.save(output_filename)
|
||||
else:
|
||||
print('Saving script model as: {}'.format(output_filename))
|
||||
container = torch.jit.script(module)
|
||||
container.save(output_filename)
|
||||
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Adapted From: https://github.com/pytorch/pytorch/issues/20356#issuecomment-545572400
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from collections import OrderedDict
|
||||
import re
|
||||
import yaml
|
||||
from math import sqrt
|
||||
|
||||
from tacotron_model import Tacotron2
|
||||
|
||||
# Going back in time, this is an old trump txlearned model:
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_ljs_arpabet_ckpt30000_2020_02_06'
|
||||
# Starting again with new samples:
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-14_chkpt7000'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-14_chkpt11500'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-14_chkpt17000'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-14_chkpt27000'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-14_chkpt27000-18000'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-14_chkpt53500'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-14_chkpt112500'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-23_chkpt8000'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-23_chkpt13000'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-24_chkpt2000'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-24_chkpt16000'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-25_chkpt7000'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-25_chkpt17000'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-25_chkpt26500'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-25_chkpt40000'
|
||||
tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_trump_txlearn_arpabet_2020-05-25_chkpt50000'
|
||||
|
||||
|
||||
#output_filename = 'tacotron_arpabet_1.pt'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_ckpt30000_2020_02_06.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-14_ckpt7000.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-14_ckpt11500.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-14_ckpt17000.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-14_ckpt27000.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-14_ckpt27000-18000.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-14_ckpt53500.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-14_chkpt112500.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-23_chkpt8000.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-23_chkpt13000.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-24_chkpt2000.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-24_chkpt16000.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-25_chkpt7000.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-25_chkpt17000.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-25_chkpt26500.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-25_chkpt40000.jit'
|
||||
output_filename = 'tacotron2_trump_txlearn_ljs_arpabet_2020-05-25_chkpt50000.jit'
|
||||
|
||||
|
||||
# NB: Tacotron's original shape is [148, 512]. Arpabet encoding is [256, 512].
|
||||
NUM_SYMBOLS_ENCODED = 256 # 148 by default, 256 with my custom Arpabet encoding
|
||||
|
||||
#torch.set_default_tensor_type('torch.DoubleTensor')
|
||||
#torch.set_default_tensor_type('torch.FloatTensor')
|
||||
|
||||
print('Loading tacotron model...')
|
||||
#tacotron_model_file = '/home/bt/models/tacotron2-nvidia/tacotron2_statedict.pt'
|
||||
tacotron_model = torch.load(tacotron_model_file, map_location=torch.device('cpu'))
|
||||
|
||||
def cuda_to_cpu(model):
|
||||
"""Recursively make everything non-CUDA"""
|
||||
if isinstance(model, dict) or isinstance(model, OrderedDict):
|
||||
for key, value in model.items():
|
||||
model[key] = cuda_to_cpu(value)
|
||||
return model
|
||||
if isinstance(model, list):
|
||||
for i, value in enumerate(model):
|
||||
model[i] = cuda_to_cpu(value)
|
||||
elif isinstance(model, torch.Tensor):
|
||||
return model.cpu()
|
||||
else:
|
||||
return model
|
||||
|
||||
# TODO: Maybe not necessary anymore with `map_location=torch.device('cpu')`
|
||||
#print('Converitng to CPU model...')
|
||||
#tacotron_model = cuda_to_cpu(tacotron_model)
|
||||
|
||||
print('Remove unused keys...')
|
||||
new_state_dict = [] # Rebuild the ordered dict
|
||||
for key, value in tacotron_model['state_dict'].items():
|
||||
if key.startswith('postnet'):
|
||||
continue
|
||||
new_state_dict.append((key, value))
|
||||
|
||||
tacotron_model['state_dict'] = OrderedDict(new_state_dict)
|
||||
|
||||
print('Containerizing model...')
|
||||
# Save arbitrary values supported by TorchScript
|
||||
# https://pytorch.org/docs/master/jit.html#supported-type
|
||||
|
||||
# Yay duck typing
|
||||
class HParams:
|
||||
mask_padding = True
|
||||
fp16_run = False
|
||||
n_mel_channels = 80
|
||||
n_frames_per_step = 1
|
||||
n_symbols = NUM_SYMBOLS_ENCODED
|
||||
symbols_embedding_dim = 512
|
||||
encoder_kernel_size = 5
|
||||
encoder_n_convolutions = 3
|
||||
encoder_embedding_dim = 512
|
||||
attention_rnn_dim = 1024
|
||||
decoder_rnn_dim = 1024
|
||||
prenet_dim = 256
|
||||
max_decoder_steps = 1000
|
||||
gate_threshold = 0.5
|
||||
p_attention_dropout = 0.1
|
||||
p_decoder_dropout = 0.1
|
||||
attention_dim = 128
|
||||
attention_location_n_filters = 32
|
||||
attention_location_kernel_size = 31
|
||||
|
||||
module = Tacotron2(HParams())
|
||||
|
||||
print('Load state dict...')
|
||||
module.load_state_dict(tacotron_model['state_dict'])
|
||||
#module.eval() # NB: Complains unless called: Did you forget call .eval() on your model?
|
||||
|
||||
# NB: Tracing evaluates the model on input and unrolls and hardcodes branching
|
||||
# and loops. Scripting allows these to remain by converting the entire program
|
||||
# to TorchScript. Scripting seems MUCH faster, is also harder get working.
|
||||
trace_model = False
|
||||
if trace_model:
|
||||
print('Tracing model and saving as:'.format(output_filename))
|
||||
text_sequence_file = '/home/bt/dev/voder/data/text/tacotron_text_sequence.pt'
|
||||
text_sequence = torch.load(text_sequence_file, map_location=torch.device('cpu'))
|
||||
# NB: Getting `Tracing failed sanity checks! Graphs differed across invocations!`
|
||||
traced_script_module = torch.jit.trace(module, text_sequence, check_trace=False)
|
||||
traced_script_module.save(output_filename)
|
||||
else:
|
||||
print('Saving script model as: {}'.format(output_filename))
|
||||
container = torch.jit.script(module)
|
||||
container.save(output_filename)
|
||||
|
||||
print('Done.')
|
||||
@@ -1,15 +0,0 @@
|
||||
[package]
|
||||
name = "tacotron_tensorflow"
|
||||
version = "0.0.1"
|
||||
authors = [ "Brandon Thomas <bt@brand.io>",
|
||||
"Brandon Thomas <echelon@gmail.com>" ]
|
||||
|
||||
[[bin]]
|
||||
name = "tacotron_tensorflow"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
hound = "3.4.0" # wav encoder/decoder
|
||||
image = "0.23.0-preview.0"
|
||||
sample = "0.10.0" # DSP toolkit
|
||||
tensorflow = "0.14.0"
|
||||
@@ -1,280 +0,0 @@
|
||||
/**
|
||||
* This takes SavedModels from Keith Ito's Tacotron impl and runs them in Rust.
|
||||
* Critically, it can short-circuit the entire model evaluation and only run the pipeline
|
||||
* pieces that we care about.
|
||||
*/
|
||||
extern crate hound;
|
||||
extern crate image;
|
||||
extern crate sample;
|
||||
extern crate tensorflow;
|
||||
|
||||
pub mod model;
|
||||
|
||||
use tensorflow::{
|
||||
SessionOptions,
|
||||
Session,
|
||||
Graph,
|
||||
Tensor,
|
||||
SessionRunArgs
|
||||
};
|
||||
|
||||
use model::print_tensorfow_version;
|
||||
use sample::ring_buffer::Slice;
|
||||
use image::RgbImage;
|
||||
use image::ImageBuffer;
|
||||
use image::Rgb;
|
||||
use image::imageops::{rotate90, flip_horizontal, flip_vertical, resize, FilterType, crop};
|
||||
|
||||
const INPUT_NAME : &'static str = "inputs";
|
||||
const INPUT_LENGTHS_NAME : &'static str = "input_lengths";
|
||||
|
||||
// NB: This goes through the griffin-lim pipeline, but is hella slow off GPU
|
||||
// The output of this is *actual* audio, though there are a few more steps in the
|
||||
// network to run before the audio is a 1:1 match with Tacotron Python.
|
||||
// Tensor("model/griffinlim/Squeeze:0", shape=(?,), dtype=float32)
|
||||
//const OUTPUT_NAME : &'static str = "model/griffinlim/Squeeze";
|
||||
|
||||
// This appears to be the mel spectrogram before griffin-lim reconstruction.
|
||||
// Image output shows visible formants. If my thinking holds, we can short circuit
|
||||
// Griffin-Lim reconstruction and send this to a trained MelGan network for a HUGE
|
||||
// performance boost end-to-end. It should even be realtime on a CPU.
|
||||
// Executing the model to this point is *extremely* fast.
|
||||
const OUTPUT_NAME : &'static str = "model/Pow_1";
|
||||
|
||||
/**
|
||||
* To save:
|
||||
print('Saving model...')
|
||||
|
||||
# https://github.com/tensorflow/models/issues/3530#issuecomment-395968881
|
||||
output_dir = './tx_trained2/'
|
||||
builder = tf.saved_model.builder.SavedModelBuilder(output_dir)
|
||||
|
||||
builder.add_meta_graph_and_variables(
|
||||
self.session,
|
||||
[tf.saved_model.tag_constants.SERVING],
|
||||
main_op=tf.tables_initializer(),
|
||||
)
|
||||
|
||||
builder.save()
|
||||
|
||||
# Save the model text labels to dereference
|
||||
definition = self.session.graph_def
|
||||
directory = 'saved_model_labels'
|
||||
tf.train.write_graph(definition, directory, 'model_labels.pb', as_text=True)
|
||||
*/
|
||||
pub fn main() {
|
||||
println!("Tacotron");
|
||||
print_tensorfow_version();
|
||||
|
||||
let dir_name = "/home/bt/dev/2nd/tacotron/tx_trained2";
|
||||
|
||||
let mut graph = Graph::new();
|
||||
let session = Session::from_saved_model(
|
||||
&SessionOptions::new(),
|
||||
&["serve"],
|
||||
&mut graph,
|
||||
dir_name,
|
||||
).expect("Should load");
|
||||
|
||||
println!("Model Loaded!");
|
||||
|
||||
// cleaned string "my name is donald trump"
|
||||
let sentence = vec![
|
||||
40, 52, 64,
|
||||
41, 28, 40, 32, 64,
|
||||
36, 46, 64,
|
||||
31, 42, 41, 28, 39, 31, 64,
|
||||
47, 45, 48, 40, 43, 1
|
||||
];
|
||||
|
||||
/*let sentence = vec![
|
||||
40, 52, 64,
|
||||
40, 52, 64,
|
||||
40, 52, 64,
|
||||
40, 52, 64,
|
||||
40, 52, 64,
|
||||
40, 52, 64,
|
||||
40, 52, 64,
|
||||
41, 28, 40, 32, 64,
|
||||
41, 28, 40, 32, 64,
|
||||
41, 28, 40, 32, 64,
|
||||
41, 28, 40, 32, 64,
|
||||
41, 28, 40, 32, 64,
|
||||
41, 28, 40, 32, 64,
|
||||
41, 28, 40, 32, 64,
|
||||
36, 46, 64,
|
||||
36, 46, 64,
|
||||
36, 46, 64,
|
||||
36, 46, 64,
|
||||
36, 46, 64,
|
||||
31, 42, 41, 28, 39, 31, 64,
|
||||
47, 45, 48, 40, 43, 1
|
||||
];*/
|
||||
|
||||
//for _i in 0..32 {
|
||||
convert_sentence(&mut graph, &session, &sentence)
|
||||
//}
|
||||
}
|
||||
|
||||
fn convert_sentence(graph: &mut Graph, session: &Session, sentence: &Vec<i32>) -> () {
|
||||
let mut input = Tensor::new(&[1, sentence.len() as u64])
|
||||
.with_values(
|
||||
// cleaned string "my name is donald trump"
|
||||
sentence.slice()
|
||||
)
|
||||
.unwrap();
|
||||
let mut input_length = Tensor::new(&[1])
|
||||
.with_values(&[sentence.len() as i32])
|
||||
.unwrap();
|
||||
println!(">>> Input tensor dims: {:?}", input.dims());
|
||||
{
|
||||
let mut args = SessionRunArgs::new();
|
||||
|
||||
println!(">>> Inputs ...");
|
||||
args.add_feed(&graph.operation_by_name_required(INPUT_NAME)
|
||||
.expect(INPUT_NAME), 0, &input);
|
||||
|
||||
println!(">>> Input Lengths ...");
|
||||
args.add_feed(&graph.operation_by_name_required(INPUT_LENGTHS_NAME)
|
||||
.expect(INPUT_LENGTHS_NAME), 0, &input_length);
|
||||
|
||||
let z = args.request_fetch(
|
||||
&graph.operation_by_name_required(OUTPUT_NAME)
|
||||
.expect(OUTPUT_NAME), 0);
|
||||
|
||||
println!(">>> Running...");
|
||||
|
||||
session.run(&mut args).expect("Run success");
|
||||
|
||||
// Check our results.
|
||||
let z_res = args.fetch::<f32>(z).expect("ret");
|
||||
|
||||
println!("z_rez.dims(): {:?}", z_res.dims());
|
||||
println!("z_rez: {:?}", z_res);
|
||||
|
||||
println!("Data[0]: {:?}", &z_res.get(0));
|
||||
println!("Data[1]: {:?}", &z_res.get(1));
|
||||
println!("Data[2]: {:?}", &z_res.get(2));
|
||||
|
||||
visualize(z_res);
|
||||
|
||||
//let processed = process_audio(z_res.to_vec());
|
||||
//write_wav(processed);
|
||||
}
|
||||
}
|
||||
|
||||
fn visualize(tensor : Tensor<f32>) {
|
||||
let dims = tensor.dims();
|
||||
let width = *dims.get(0).unwrap() as u32;
|
||||
let height = *dims.get(1).unwrap() as u32;
|
||||
|
||||
let mut image: RgbImage = ImageBuffer::new(width, height);
|
||||
|
||||
let mut i: u32 = 0;
|
||||
let mut j: u32 = 0;
|
||||
let mut k: u32 = 0;
|
||||
|
||||
for x in tensor.iter() {
|
||||
if j >= height {
|
||||
j = 0;
|
||||
i += 1
|
||||
}
|
||||
if i >= width {
|
||||
println!("Break @ {}", k);
|
||||
break;
|
||||
}
|
||||
/*if i % 5 == 0 {
|
||||
image.put_pixel(i, j, Rgb([0, 0, 0]));
|
||||
} else {
|
||||
image.put_pixel(i, j, Rgb([0, 255, 0]));
|
||||
}*/
|
||||
// [-1.0, 1.0]
|
||||
let val = (*x) * 255f32;
|
||||
let val = val.abs() as u8;
|
||||
image.put_pixel(i, j, Rgb([val, val, val]));
|
||||
j += 1;
|
||||
k += 1;
|
||||
}
|
||||
|
||||
//image = rotate90(&image);
|
||||
//image = flip_horizontal(&image);
|
||||
image = flip_vertical(&image);
|
||||
let cropped = crop(&mut image,0, 0, 200, height);
|
||||
let mut image = cropped.to_image();
|
||||
image = resize(&image, 200, 100, FilterType::Triangle);
|
||||
image.save("output.png").unwrap();
|
||||
|
||||
println!("Dimensions: {:?}", dims);
|
||||
println!("Width: {:?}", width);
|
||||
println!("Height: {:?}", height);
|
||||
println!("Num pixels calc: {:?}", (width * height));
|
||||
println!("Num pixels covered: {:?}", k);
|
||||
}
|
||||
|
||||
fn process_audio(signal: Vec<f32>) -> Vec<f64> {
|
||||
// wav *= 32767 / max(0.01, np.max(np.abs(wav)))
|
||||
// scipy.io.wavfile.write(path, hparams.sample_rate, wav.astype(np.int16))
|
||||
|
||||
// abs
|
||||
//[0.0000000e+00 4.1254760e-12 7.1285297e-11 ... 5.9092813e-14 4.0335224e-14
|
||||
// 2.1358613e-15]
|
||||
//max abs
|
||||
//0.08918631
|
||||
|
||||
let signal = signal.iter()
|
||||
.map(|f| *f as f64)
|
||||
.collect::<Vec<f64>>();
|
||||
|
||||
let abs = signal.iter()
|
||||
.map(|f| f.abs())
|
||||
.collect::<Vec<f64>>();
|
||||
|
||||
println!("Abs[0]: {:?}", &abs.get(0));
|
||||
println!("Abs[1]: {:?}", &abs.get(1));
|
||||
println!("Abs[2]: {:?}", &abs.get(2));
|
||||
|
||||
let max : f64 = abs.iter()
|
||||
.fold(0.01f64, |x, y| x.max(*y));
|
||||
|
||||
|
||||
let signal_mult = signal.iter()
|
||||
.map(|f| *f * 32767.0f64)
|
||||
.collect::<Vec<f64>>();
|
||||
|
||||
let signal_div = signal_mult.iter()
|
||||
.map(|f| *f / max)
|
||||
.collect::<Vec<f64>>();
|
||||
|
||||
println!("Max: {:?}", max);
|
||||
|
||||
let mult = 32767.0f64 / max;
|
||||
let rust_mult = 367225.3669259117;
|
||||
let python_mult = 367399.4329151011;
|
||||
|
||||
println!("Mult: {:?}", mult);
|
||||
|
||||
/*let result = signal.iter()
|
||||
.map(|f| f * python_mult)
|
||||
.collect::<Vec<f64>>();
|
||||
|
||||
println!("Res[0]: {:?}", &abs.get(0));
|
||||
println!("Res[1]: {:?}", &abs.get(1));
|
||||
println!("Res[2]: {:?}", &abs.get(2));*/
|
||||
|
||||
signal_div
|
||||
}
|
||||
|
||||
fn write_wav(signal: Vec<f64>) {
|
||||
let spec = hound::WavSpec {
|
||||
channels: 1,
|
||||
sample_rate: 16000,
|
||||
bits_per_sample: 32,
|
||||
sample_format: hound::SampleFormat::Float,
|
||||
};
|
||||
let mut writer = hound::WavWriter::create("output.wav", spec).unwrap();
|
||||
|
||||
for sample in signal {
|
||||
writer.write_sample(sample as f32).unwrap();
|
||||
}
|
||||
writer.finalize().unwrap();
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
use tensorflow::Code;
|
||||
use tensorflow::Graph;
|
||||
use tensorflow::ImportGraphDefOptions;
|
||||
use tensorflow::Session;
|
||||
use tensorflow::SessionOptions;
|
||||
use tensorflow::SessionRunArgs;
|
||||
use tensorflow::Status;
|
||||
use tensorflow::Tensor;
|
||||
use tensorflow::version;
|
||||
|
||||
const INPUT_NAME : &'static str = "input_A_test";
|
||||
const OUTPUT_NAME : &'static str = "generator_A2B_3/output_transpose";
|
||||
|
||||
pub struct VoiceModel {
|
||||
graph: Graph,
|
||||
session: Session,
|
||||
}
|
||||
|
||||
impl VoiceModel {
|
||||
pub fn load(dir_name: &str) -> Self {
|
||||
let mut graph = Graph::new();
|
||||
let session = Session::from_saved_model(
|
||||
&SessionOptions::new(),
|
||||
&["serve"],
|
||||
&mut graph,
|
||||
dir_name,
|
||||
).expect("Should load");
|
||||
|
||||
VoiceModel {
|
||||
graph,
|
||||
session,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn evaluate(&self, input: &Tensor<f32>) {
|
||||
println!(">>> Input tensor dims: {:?}", input.dims());
|
||||
|
||||
// Run the graph.
|
||||
let mut args = SessionRunArgs::new();
|
||||
|
||||
// input_A_test:
|
||||
// Tensor("input_A_test:0", shape=(?, 24, ?), dtype=float32)
|
||||
args.add_feed(&self.graph.operation_by_name_required(INPUT_NAME)
|
||||
.expect(INPUT_NAME), 0, &input);
|
||||
|
||||
// generation_B_test:
|
||||
// Tensor("generator_A2B_3/output_transpose:0", shape=(?, 24, ?), dtype=float32)
|
||||
let z = args.request_fetch(
|
||||
&self.graph.operation_by_name_required(OUTPUT_NAME)
|
||||
.expect(OUTPUT_NAME), 0);
|
||||
|
||||
self.session.run(&mut args).expect("Run success");
|
||||
|
||||
// Check our results.
|
||||
let z_res = args.fetch::<f32>(z).expect("ret");
|
||||
|
||||
println!("z_rez.dims(): {:?}", z_res.dims());
|
||||
println!("z_rez: {:?}", z_res);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load_model() {
|
||||
let model_dir = "/home/bt/dev/voder/saved_model";
|
||||
let model = VoiceModel::load(model_dir);
|
||||
|
||||
// ] generation_B_test: Tensor("generator_A2B_3/output_transpose:0", shape=(?, 24, ?), dtype=float32)
|
||||
// ] input_A_test: Tensor("input_A_test:0", shape=(?, 24, ?), dtype=float32)
|
||||
// ] inputs.shape: (1, 24, 380) -- the last dimension is the signal
|
||||
|
||||
let mut input = Tensor::new(&[1, 24, 10])
|
||||
.with_values(&[
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
model.evaluate(&input);
|
||||
model.evaluate(&input);
|
||||
model.evaluate(&input);
|
||||
}
|
||||
|
||||
/*fn load_model_2() {
|
||||
// from regression_checkpoint.rs example
|
||||
|
||||
let filename = "/home/bt/dev/voder/saved_model/saved_model.pb";
|
||||
|
||||
let mut graph = Graph::new();
|
||||
let mut proto = Vec::new();
|
||||
|
||||
File::open(filename)
|
||||
.expect("opened")
|
||||
.read_to_end(&mut proto)
|
||||
.expect("cannot read");
|
||||
|
||||
graph.import_graph_def(&proto, &ImportGraphDefOptions::new()).expect("cannot import");
|
||||
|
||||
let session = Session::new(&SessionOptions::new(), &graph).expect("cannot session");
|
||||
|
||||
/*let op_x = graph.operation_by_name_required("x").expect("x");
|
||||
let op_y = graph.operation_by_name_required("y").expect("y");
|
||||
let op_init = graph.operation_by_name_required("init").expect("init");
|
||||
let op_train = graph.operation_by_name_required("train").expect("train");
|
||||
let op_w = graph.operation_by_name_required("w").expect("w");
|
||||
let op_b = graph.operation_by_name_required("b").expect("b");
|
||||
let op_file_path = graph.operation_by_name_required("save/Const").expect("const");
|
||||
let op_save = graph.operation_by_name_required("save/control_dependency").expect("save");*/
|
||||
|
||||
//let file_path_tensor: Tensor<String> =
|
||||
// Tensor::from(String::from("/home/bt/dev/voder/extra_model/sf1_tm1.ckpt.data-00000-of-00001"));
|
||||
//println!("Tensor: {:?}", file_path_tensor);
|
||||
}*/
|
||||
|
||||
pub fn print_tensorfow_version() {
|
||||
// Python TensorFlow version: 1.14.0
|
||||
// Rust TensorFlow version: 1.13.1 (default, must be hand-upgraded)
|
||||
let version = version().expect("version");
|
||||
println!("Tensorflow version: {}", version);
|
||||
|
||||
let graph = Graph::new();
|
||||
let options = SessionOptions::new();
|
||||
let session = Session::new(&options, &graph).unwrap();
|
||||
let devices = session.device_list().unwrap();
|
||||
for device in devices {
|
||||
println!("Device: {:?}", device);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
[package]
|
||||
name = "voice_conversion_sidecar"
|
||||
version = "0.1.0"
|
||||
authors = [ "Brandon Thomas <bt@brand.io>",
|
||||
"Brandon Thomas <echelon@gmail.com>" ]
|
||||
[[bin]]
|
||||
name = "voice_converter_sidecar"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
byteorder = "1.3.2" # byte-level operations
|
||||
bytes = "0.4.12" # required by 'prost'
|
||||
cpal = "0.10.0" # hardware audio out
|
||||
failure = "0.1.5" # required by cpal
|
||||
prost = "0.5.*" # protobuf
|
||||
prost-types = "0.5.0" # protobuf
|
||||
wavy = "0.1.2" # real time audio capture and playback
|
||||
# TODO: Move these into the voder project workspace.
|
||||
world-sys = { path = "/home/bt/dev/world-sys" } # AWESOME WORLD VOCODER BINDINGS HAH HA HAH
|
||||
zmq = "0.9.2" # zeromq
|
||||
|
||||
[build-dependencies]
|
||||
prost-build = "0.5.0"
|
||||
@@ -1,34 +0,0 @@
|
||||
Voice Converter Sidecar
|
||||
=======================
|
||||
This project spins up a Rust binary to handle audio input and output, then
|
||||
ships microphone audio over proto/zeromq to a Python sidecar running
|
||||
CycleGAN-VC to convert it into target speech.
|
||||
|
||||
### CycleGAN
|
||||
I've included a pared down copy of CycleGAN for the purposes of model
|
||||
evaluation. It handles all of the audio buffering / sidecar integration
|
||||
pieces. [The original CycleGAN I used is here](https://github.com/leimao/Voice_Converter_CycleGAN).
|
||||
|
||||
#### Note on Training
|
||||
It's possible to train and evaluate at the same time using dual GPUs
|
||||
(at least my dual 1080Ti setup). To run the sidecar on a particular
|
||||
GPU (0-indexded), use:
|
||||
|
||||
```
|
||||
CUDA_VISIBLE_DEVICES=1 ./sidecar.py
|
||||
```
|
||||
|
||||
### Proto Compilation (for Python)
|
||||
Codegen for Rust is built in. Codegen for Python uses,
|
||||
|
||||
```
|
||||
protoc --python_out=cycle_gan protos/audio.proto
|
||||
```
|
||||
|
||||
### Current Results
|
||||
Currently there is 4.39 seconds of delay between speaking and generated output
|
||||
with the sidecar setup on my desktop computer. This is really great and seems
|
||||
promising.
|
||||
|
||||
This gets up to 6.0 seconds later. Drift continues to accrue, but it's a slow
|
||||
build.
|
||||
@@ -1,11 +0,0 @@
|
||||
extern crate prost_build;
|
||||
|
||||
fn main() {
|
||||
//prost_build::compile_protos(&["protos/audio.proto"],
|
||||
// &["protos/"]).unwrap();
|
||||
prost_build::Config::new()
|
||||
.out_dir("src/protos/")
|
||||
.compile_protos(&["protos/audio.proto"],
|
||||
&["protos/"])
|
||||
.unwrap();
|
||||
}
|
||||
@@ -1,278 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Starts a demo HTTP server to capture and transform audio
|
||||
as a live demonstration of the trained model.
|
||||
|
||||
Brandon Thomas 2019-07-29 <bt@brand.io> <echelon@gmail.com>
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import falcon
|
||||
import io
|
||||
import librosa
|
||||
import numpy as np
|
||||
import os
|
||||
import scipy
|
||||
import soundfile
|
||||
import tensorflow as tf
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from falcon_multipart.middleware import MultipartMiddleware
|
||||
from model import CycleGAN
|
||||
from preprocess import *
|
||||
from wsgiref import simple_server
|
||||
|
||||
print("TensorFlow version: {}".format(tf.version.VERSION))
|
||||
|
||||
INDEX_HTML = '''
|
||||
<!doctype html>
|
||||
<html>
|
||||
<body>
|
||||
<h1>Voice Demo</h1>
|
||||
<script src="./script/recorder.js" type="application/javascript"></script>
|
||||
<script type="application/javascript">
|
||||
var audio_context;
|
||||
var recorder;
|
||||
|
||||
// NB: Capture code originates from 'recorder.js' demo.
|
||||
function startUserMedia(stream) {
|
||||
var input = audio_context.createMediaStreamSource(stream);
|
||||
window.recorder = new Recorder(input);
|
||||
}
|
||||
|
||||
function createDownloadLink() {
|
||||
window.recorder && window.recorder.exportWAV(function(blob) {
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.responseType = 'blob';
|
||||
xhr.onload=function(e) {
|
||||
console.log('response received');
|
||||
var objectUrl = window.URL.createObjectURL(this.response);
|
||||
console.log('audio URL', objectUrl);
|
||||
var audio = document.getElementById('audio');
|
||||
audio.src = objectUrl;
|
||||
audio.play();
|
||||
};
|
||||
|
||||
var fd = new FormData();
|
||||
fd.append("audio_data", blob, "audio_file.wav");
|
||||
console.log(blob);
|
||||
console.log('sending...');
|
||||
xhr.open("POST","/upload",true);
|
||||
xhr.send(fd);
|
||||
console.log('sent');
|
||||
});
|
||||
}
|
||||
|
||||
function startRecording(button) {
|
||||
window.recorder && window.recorder.record();
|
||||
document.getElementById('start').disabled = true;
|
||||
document.getElementById('stop').disabled = false;
|
||||
}
|
||||
|
||||
function stopRecording(button) {
|
||||
window.recorder && window.recorder.stop();
|
||||
document.getElementById('start').disabled = false;
|
||||
document.getElementById('stop').disabled = true;
|
||||
|
||||
createDownloadLink();
|
||||
window.recorder.clear();
|
||||
}
|
||||
|
||||
window.onload = function init() {
|
||||
try {
|
||||
audio_context = new AudioContext();
|
||||
} catch (e) {
|
||||
alert('No web audio support in this browser!');
|
||||
}
|
||||
|
||||
var media = navigator.mediaDevices.getUserMedia({ audio: true, video: false });
|
||||
window.media = media;
|
||||
console.log('media', media);
|
||||
media.then(startUserMedia);
|
||||
};
|
||||
</script>
|
||||
|
||||
<button id="start" onclick="startRecording(this);">record</button>
|
||||
<br>
|
||||
<br>
|
||||
<button id="stop" onclick="stopRecording(this);" disabled>stop</button>
|
||||
<br>
|
||||
<br>
|
||||
<audio controls id="audio">
|
||||
<source id="source" src="" type="audio/wav">
|
||||
Your browser does not support the audio element.
|
||||
</audio>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
|
||||
|
||||
class Converter():
|
||||
def __init__(self, model_dir, model_name):
|
||||
self.num_features = 24
|
||||
self.sampling_rate = 16000
|
||||
self.frame_period = 5.0
|
||||
|
||||
self.model = CycleGAN(num_features = self.num_features, mode = 'test')
|
||||
|
||||
self.model.load(filepath = os.path.join(model_dir, model_name))
|
||||
|
||||
"""
|
||||
# NB: Save the graph
|
||||
definition = self.model.sess.graph_def
|
||||
directory = 'saved_model_2'
|
||||
tf.train.write_graph(definition, directory, 'saved_model_2.pb', as_text=True)
|
||||
|
||||
# https://github.com/tensorflow/models/issues/3530#issuecomment-395968881
|
||||
output_dir = './saved_model/'
|
||||
builder = tf.saved_model.builder.SavedModelBuilder(output_dir)
|
||||
|
||||
builder.add_meta_graph_and_variables(
|
||||
self.model.sess,
|
||||
[tf.saved_model.tag_constants.SERVING],
|
||||
main_op=tf.tables_initializer(),
|
||||
)
|
||||
|
||||
builder.save()
|
||||
"""
|
||||
|
||||
"""
|
||||
builder.add_meta_graph_and_variables(
|
||||
self.model.sess,
|
||||
[tf.saved_model.tag_constants.SERVING],
|
||||
signature_def_map={
|
||||
'predict_images':
|
||||
prediction_signature,
|
||||
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
|
||||
classification_signature,
|
||||
},
|
||||
main_op=tf.tables_initializer())
|
||||
"""
|
||||
|
||||
self.mcep_normalization_params = np.load(os.path.join(model_dir, 'mcep_normalization.npz'))
|
||||
self.mcep_mean_A = self.mcep_normalization_params['mean_A']
|
||||
self.mcep_std_A = self.mcep_normalization_params['std_A']
|
||||
self.mcep_mean_B = self.mcep_normalization_params['mean_B']
|
||||
self.mcep_std_B = self.mcep_normalization_params['std_B']
|
||||
|
||||
self.logf0s_normalization_params = np.load(os.path.join(model_dir, 'logf0s_normalization.npz'))
|
||||
self.logf0s_mean_A = self.logf0s_normalization_params['mean_A']
|
||||
self.logf0s_std_A = self.logf0s_normalization_params['std_A']
|
||||
self.logf0s_mean_B = self.logf0s_normalization_params['mean_B']
|
||||
self.logf0s_std_B = self.logf0s_normalization_params['std_B']
|
||||
|
||||
def convert(self, wav, conversion_direction='A2B'):
|
||||
wav = wav_padding(wav = wav, sr = self.sampling_rate, frame_period = self.frame_period, multiple = 4)
|
||||
f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = self.sampling_rate, frame_period = self.frame_period)
|
||||
coded_sp = world_encode_spectral_envelop(sp = sp, fs = self.sampling_rate, dim = self.num_features)
|
||||
coded_sp_transposed = coded_sp.T
|
||||
|
||||
if conversion_direction == 'A2B':
|
||||
f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_A, std_log_src = self.logf0s_std_A, mean_log_target = self.logf0s_mean_B, std_log_target = self.logf0s_std_B)
|
||||
coded_sp_norm = (coded_sp_transposed - self.mcep_mean_A) / self.mcep_std_A
|
||||
coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0]
|
||||
coded_sp_converted = coded_sp_converted_norm * self.mcep_std_B + self.mcep_mean_B
|
||||
else:
|
||||
f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_B, std_log_src = self.logf0s_std_B, mean_log_target = self.logf0s_mean_A, std_log_target = self.logf0s_std_A)
|
||||
coded_sp_norm = (coded_sp_transposed - self.mcep_mean_B) / self.mcep_std_B
|
||||
coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0]
|
||||
coded_sp_converted = coded_sp_converted_norm * self.mcep_std_A + self.mcep_mean_A
|
||||
|
||||
coded_sp_converted = coded_sp_converted.T
|
||||
coded_sp_converted = np.ascontiguousarray(coded_sp_converted)
|
||||
decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, fs = self.sampling_rate)
|
||||
wav_transformed = world_speech_synthesis(f0 = f0_converted, decoded_sp = decoded_sp_converted, ap = ap, fs = self.sampling_rate, frame_period = self.frame_period)
|
||||
|
||||
# For debugging model output, uncomment the following line:
|
||||
# librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate)
|
||||
|
||||
# TODO: Perhaps ditch this. It's probably unnecessary work.
|
||||
upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000)
|
||||
pcm_data = upsampled.astype(np.float64)
|
||||
stereo_pcm_data = np.tile(pcm_data, (2,1)).T
|
||||
|
||||
buf = io.BytesIO()
|
||||
scipy.io.wavfile.write(buf, 48000, stereo_pcm_data.astype(np.float32))
|
||||
return buf
|
||||
|
||||
# Set up model
|
||||
# This should live long in memory, so we do it up front.
|
||||
model_dir_default = './model/sf1_tm1'
|
||||
model_name_default = 'sf1_tm1.ckpt'
|
||||
converter = Converter(model_dir_default, model_name_default)
|
||||
|
||||
class IndexHandler():
|
||||
def on_get(self, request, response):
|
||||
response.content_type = 'text/html'
|
||||
response.body = INDEX_HTML
|
||||
|
||||
class ApiHandler():
|
||||
def on_post(self, request, response):
|
||||
# NB: uses middleware to pull out data.
|
||||
form_data = request.params['audio_data'].file
|
||||
data, samplerate = soundfile.read(form_data)
|
||||
|
||||
print('samplerate', samplerate)
|
||||
print('data', data)
|
||||
print('data.shape', data.shape)
|
||||
print('data.dtype', data.dtype)
|
||||
|
||||
# For debugging browser input, uncomment the following line:
|
||||
# scipy.io.wavfile.write('browser_input_audio.wav', samplerate, data)
|
||||
|
||||
# NB: Convert the input stereo signal into mono.
|
||||
# In the future the frontend should be responsible for sampling details.
|
||||
mono = data[:, 0]
|
||||
print('mono', mono)
|
||||
print('mono.shape', mono.shape)
|
||||
print('mono.dtype', mono.dtype)
|
||||
|
||||
# NB: We must downsample to the rate that the network is trained on.
|
||||
downsampled = librosa.resample(mono, samplerate, 16000)
|
||||
print('downsampled', downsampled)
|
||||
print('downsampled.shape', downsampled.shape)
|
||||
print('downsampled.dtype', downsampled.dtype)
|
||||
|
||||
# Evaluate the model
|
||||
print(">>> Converting...")
|
||||
results = converter.convert(downsampled, conversion_direction = 'A2B')
|
||||
|
||||
print('type(results)', type(results))
|
||||
|
||||
temp_dir = tempfile.TemporaryDirectory(prefix='tmp_ml_audio')
|
||||
temp_file = tempfile.NamedTemporaryFile(suffix='.wav')
|
||||
|
||||
temp_file.write(results.read())
|
||||
|
||||
out_file = temp_dir.name + '/output.ogg'
|
||||
|
||||
# NB: Browsers have a great deal of trouble decoding WAV files unless they are in the
|
||||
# narrow slice of the WAV spec expected. None of the {librosa, scipy, soundfile} python
|
||||
# tools do a good job of this, so here we shell out to ffmpeg and generate OGG.
|
||||
# It's lazy and messy, but it works for now.
|
||||
# See https://github.com/librosa/librosa/issues/361 for a survey of the library landscape
|
||||
# See https://bugzilla.mozilla.org/show_bug.cgi?id=523837 for one of dozens of browser codec bugs
|
||||
_stdout = subprocess.check_output(['ffmpeg', '-i', temp_file.name, '-acodec', 'libvorbis', out_file])
|
||||
|
||||
response.content_type = 'audio/ogg'
|
||||
with open(out_file, mode='rb') as f:
|
||||
response.data = f.read()
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--port', type=int, default=8000)
|
||||
args = parser.parse_args()
|
||||
|
||||
api = falcon.API(middleware=[MultipartMiddleware()])
|
||||
api.add_route('/', IndexHandler())
|
||||
api.add_route('/upload', ApiHandler())
|
||||
api.add_static_route('/script', os.path.abspath('./script'))
|
||||
api.add_static_route('/sound', os.path.abspath('./sound'))
|
||||
print('Serving on 0.0.0.0:%d' % args.port)
|
||||
simple_server.make_server('0.0.0.0', args.port, api).serve_forever()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@@ -1,183 +0,0 @@
|
||||
import os
|
||||
import tensorflow as tf
|
||||
from module import discriminator, generator_gatedcnn
|
||||
from utils import l1_loss, l2_loss, cross_entropy_loss
|
||||
from datetime import datetime
|
||||
|
||||
class CycleGAN(object):
|
||||
|
||||
def __init__(self, num_features, discriminator = discriminator, generator = generator_gatedcnn, mode = 'train', log_dir = './log'):
|
||||
|
||||
self.num_features = num_features
|
||||
self.input_shape = [None, num_features, None] # [batch_size, num_features, num_frames]
|
||||
|
||||
self.discriminator = discriminator
|
||||
self.generator = generator
|
||||
self.mode = mode
|
||||
|
||||
self.build_model()
|
||||
self.optimizer_initializer()
|
||||
|
||||
self.saver = tf.train.Saver()
|
||||
|
||||
#config = tf.ConfigProto(device_count = {'GPU': 1})
|
||||
#self.sess = tf.Session(config=config)
|
||||
self.sess = tf.Session()
|
||||
self.sess.run(tf.global_variables_initializer())
|
||||
|
||||
if self.mode == 'train':
|
||||
self.train_step = 0
|
||||
now = datetime.now()
|
||||
self.log_dir = os.path.join(log_dir, now.strftime('%Y%m%d-%H%M%S'))
|
||||
self.writer = tf.summary.FileWriter(self.log_dir, tf.get_default_graph())
|
||||
self.generator_summaries, self.discriminator_summaries = self.summary()
|
||||
|
||||
def build_model(self):
|
||||
|
||||
# Placeholders for real training samples
|
||||
self.input_A_real = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_A_real')
|
||||
self.input_B_real = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_B_real')
|
||||
# Placeholders for fake generated samples
|
||||
self.input_A_fake = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_A_fake')
|
||||
self.input_B_fake = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_B_fake')
|
||||
# Placeholder for test samples
|
||||
self.input_A_test = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_A_test')
|
||||
self.input_B_test = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_B_test')
|
||||
|
||||
self.generation_B = self.generator(inputs = self.input_A_real, reuse = False, scope_name = 'generator_A2B')
|
||||
self.cycle_A = self.generator(inputs = self.generation_B, reuse = False, scope_name = 'generator_B2A')
|
||||
|
||||
self.generation_A = self.generator(inputs = self.input_B_real, reuse = True, scope_name = 'generator_B2A')
|
||||
self.cycle_B = self.generator(inputs = self.generation_A, reuse = True, scope_name = 'generator_A2B')
|
||||
|
||||
self.generation_A_identity = self.generator(inputs = self.input_A_real, reuse = True, scope_name = 'generator_B2A')
|
||||
self.generation_B_identity = self.generator(inputs = self.input_B_real, reuse = True, scope_name = 'generator_A2B')
|
||||
|
||||
self.discrimination_A_fake = self.discriminator(inputs = self.generation_A, reuse = False, scope_name = 'discriminator_A')
|
||||
self.discrimination_B_fake = self.discriminator(inputs = self.generation_B, reuse = False, scope_name = 'discriminator_B')
|
||||
|
||||
# Cycle loss
|
||||
self.cycle_loss = l1_loss(y = self.input_A_real, y_hat = self.cycle_A) + l1_loss(y = self.input_B_real, y_hat = self.cycle_B)
|
||||
|
||||
# Identity loss
|
||||
self.identity_loss = l1_loss(y = self.input_A_real, y_hat = self.generation_A_identity) + l1_loss(y = self.input_B_real, y_hat = self.generation_B_identity)
|
||||
|
||||
# Place holder for lambda_cycle and lambda_identity
|
||||
self.lambda_cycle = tf.placeholder(tf.float32, None, name = 'lambda_cycle')
|
||||
self.lambda_identity = tf.placeholder(tf.float32, None, name = 'lambda_identity')
|
||||
|
||||
# Generator loss
|
||||
# Generator wants to fool discriminator
|
||||
self.generator_loss_A2B = l2_loss(y = tf.ones_like(self.discrimination_B_fake), y_hat = self.discrimination_B_fake)
|
||||
self.generator_loss_B2A = l2_loss(y = tf.ones_like(self.discrimination_A_fake), y_hat = self.discrimination_A_fake)
|
||||
|
||||
# Merge the two generators and the cycle loss
|
||||
self.generator_loss = self.generator_loss_A2B + self.generator_loss_B2A + self.lambda_cycle * self.cycle_loss + self.lambda_identity * self.identity_loss
|
||||
|
||||
# Discriminator loss
|
||||
self.discrimination_input_A_real = self.discriminator(inputs = self.input_A_real, reuse = True, scope_name = 'discriminator_A')
|
||||
self.discrimination_input_B_real = self.discriminator(inputs = self.input_B_real, reuse = True, scope_name = 'discriminator_B')
|
||||
self.discrimination_input_A_fake = self.discriminator(inputs = self.input_A_fake, reuse = True, scope_name = 'discriminator_A')
|
||||
self.discrimination_input_B_fake = self.discriminator(inputs = self.input_B_fake, reuse = True, scope_name = 'discriminator_B')
|
||||
|
||||
# Discriminator wants to classify real and fake correctly
|
||||
self.discriminator_loss_input_A_real = l2_loss(y = tf.ones_like(self.discrimination_input_A_real), y_hat = self.discrimination_input_A_real)
|
||||
self.discriminator_loss_input_A_fake = l2_loss(y = tf.zeros_like(self.discrimination_input_A_fake), y_hat = self.discrimination_input_A_fake)
|
||||
self.discriminator_loss_A = (self.discriminator_loss_input_A_real + self.discriminator_loss_input_A_fake) / 2
|
||||
|
||||
self.discriminator_loss_input_B_real = l2_loss(y = tf.ones_like(self.discrimination_input_B_real), y_hat = self.discrimination_input_B_real)
|
||||
self.discriminator_loss_input_B_fake = l2_loss(y = tf.zeros_like(self.discrimination_input_B_fake), y_hat = self.discrimination_input_B_fake)
|
||||
self.discriminator_loss_B = (self.discriminator_loss_input_B_real + self.discriminator_loss_input_B_fake) / 2
|
||||
|
||||
# Merge the two discriminators into one
|
||||
self.discriminator_loss = self.discriminator_loss_A + self.discriminator_loss_B
|
||||
|
||||
# Categorize variables because we have to optimize the two sets of the variables separately
|
||||
trainable_variables = tf.trainable_variables()
|
||||
self.discriminator_vars = [var for var in trainable_variables if 'discriminator' in var.name]
|
||||
self.generator_vars = [var for var in trainable_variables if 'generator' in var.name]
|
||||
#for var in t_vars: print(var.name)
|
||||
|
||||
# Reserved for test
|
||||
self.generation_B_test = self.generator(inputs = self.input_A_test, reuse = True, scope_name = 'generator_A2B')
|
||||
self.generation_A_test = self.generator(inputs = self.input_B_test, reuse = True, scope_name = 'generator_B2A')
|
||||
|
||||
|
||||
def optimizer_initializer(self):
|
||||
|
||||
self.generator_learning_rate = tf.placeholder(tf.float32, None, name = 'generator_learning_rate')
|
||||
self.discriminator_learning_rate = tf.placeholder(tf.float32, None, name = 'discriminator_learning_rate')
|
||||
self.discriminator_optimizer = tf.train.AdamOptimizer(learning_rate = self.discriminator_learning_rate, beta1 = 0.5).minimize(self.discriminator_loss, var_list = self.discriminator_vars)
|
||||
self.generator_optimizer = tf.train.AdamOptimizer(learning_rate = self.generator_learning_rate, beta1 = 0.5).minimize(self.generator_loss, var_list = self.generator_vars)
|
||||
|
||||
def train(self, input_A, input_B, lambda_cycle, lambda_identity, generator_learning_rate, discriminator_learning_rate):
|
||||
|
||||
generation_A, generation_B, generator_loss, _, generator_summaries = self.sess.run(
|
||||
[self.generation_A, self.generation_B, self.generator_loss, self.generator_optimizer, self.generator_summaries], \
|
||||
feed_dict = {self.lambda_cycle: lambda_cycle, self.lambda_identity: lambda_identity, self.input_A_real: input_A, self.input_B_real: input_B, self.generator_learning_rate: generator_learning_rate})
|
||||
|
||||
self.writer.add_summary(generator_summaries, self.train_step)
|
||||
|
||||
discriminator_loss, _, discriminator_summaries = self.sess.run([self.discriminator_loss, self.discriminator_optimizer, self.discriminator_summaries], \
|
||||
feed_dict = {self.input_A_real: input_A, self.input_B_real: input_B, self.discriminator_learning_rate: discriminator_learning_rate, self.input_A_fake: generation_A, self.input_B_fake: generation_B})
|
||||
|
||||
self.writer.add_summary(discriminator_summaries, self.train_step)
|
||||
|
||||
self.train_step += 1
|
||||
|
||||
return generator_loss, discriminator_loss
|
||||
|
||||
|
||||
def test(self, inputs, direction):
|
||||
#print(">>> model.test()!")
|
||||
|
||||
if direction == 'A2B':
|
||||
#print("generation_B_test: {}".format(self.generation_B_test))
|
||||
#print("input_A_test: {}".format(self.input_A_test))
|
||||
generation = self.sess.run(self.generation_B_test, feed_dict = {self.input_A_test: inputs})
|
||||
elif direction == 'B2A':
|
||||
#print("generation_A_test: {}".format(self.generation_A_test))
|
||||
#print("input_B_test: {}".format(self.input_B_test))
|
||||
generation = self.sess.run(self.generation_A_test, feed_dict = {self.input_B_test: inputs})
|
||||
else:
|
||||
raise Exception('Conversion direction must be specified.')
|
||||
|
||||
return generation
|
||||
|
||||
|
||||
def save(self, directory, filename):
|
||||
|
||||
if not os.path.exists(directory):
|
||||
os.makedirs(directory)
|
||||
self.saver.save(self.sess, os.path.join(directory, filename))
|
||||
|
||||
return os.path.join(directory, filename)
|
||||
|
||||
def load(self, filepath):
|
||||
|
||||
self.saver.restore(self.sess, filepath)
|
||||
|
||||
|
||||
def summary(self):
|
||||
|
||||
with tf.name_scope('generator_summaries'):
|
||||
cycle_loss_summary = tf.summary.scalar('cycle_loss', self.cycle_loss)
|
||||
identity_loss_summary = tf.summary.scalar('identity_loss', self.identity_loss)
|
||||
generator_loss_A2B_summary = tf.summary.scalar('generator_loss_A2B', self.generator_loss_A2B)
|
||||
generator_loss_B2A_summary = tf.summary.scalar('generator_loss_B2A', self.generator_loss_B2A)
|
||||
generator_loss_summary = tf.summary.scalar('generator_loss', self.generator_loss)
|
||||
generator_summaries = tf.summary.merge([cycle_loss_summary, identity_loss_summary, generator_loss_A2B_summary, generator_loss_B2A_summary, generator_loss_summary])
|
||||
|
||||
with tf.name_scope('discriminator_summaries'):
|
||||
discriminator_loss_A_summary = tf.summary.scalar('discriminator_loss_A', self.discriminator_loss_A)
|
||||
discriminator_loss_B_summary = tf.summary.scalar('discriminator_loss_B', self.discriminator_loss_B)
|
||||
discriminator_loss_summary = tf.summary.scalar('discriminator_loss', self.discriminator_loss)
|
||||
discriminator_summaries = tf.summary.merge([discriminator_loss_A_summary, discriminator_loss_B_summary, discriminator_loss_summary])
|
||||
|
||||
return generator_summaries, discriminator_summaries
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
model = CycleGAN(num_features = 24)
|
||||
#print('Graph Compile Successeded.')
|
||||
@@ -1,214 +0,0 @@
|
||||
import tensorflow as tf
|
||||
|
||||
def gated_linear_layer(inputs, gates, name = None):
|
||||
|
||||
activation = tf.multiply(x = inputs, y = tf.sigmoid(gates), name = name)
|
||||
|
||||
return activation
|
||||
|
||||
def instance_norm_layer(
|
||||
inputs,
|
||||
epsilon = 1e-06,
|
||||
activation_fn = None,
|
||||
name = None):
|
||||
|
||||
instance_norm_layer = tf.contrib.layers.instance_norm(
|
||||
inputs = inputs,
|
||||
epsilon = epsilon,
|
||||
activation_fn = activation_fn)
|
||||
|
||||
return instance_norm_layer
|
||||
|
||||
def conv1d_layer(
|
||||
inputs,
|
||||
filters,
|
||||
kernel_size,
|
||||
strides = 1,
|
||||
padding = 'same',
|
||||
activation = None,
|
||||
kernel_initializer = None,
|
||||
name = None):
|
||||
|
||||
conv_layer = tf.layers.conv1d(
|
||||
inputs = inputs,
|
||||
filters = filters,
|
||||
kernel_size = kernel_size,
|
||||
strides = strides,
|
||||
padding = padding,
|
||||
activation = activation,
|
||||
kernel_initializer = kernel_initializer,
|
||||
name = name)
|
||||
|
||||
return conv_layer
|
||||
|
||||
def conv2d_layer(
|
||||
inputs,
|
||||
filters,
|
||||
kernel_size,
|
||||
strides,
|
||||
padding = 'same',
|
||||
activation = None,
|
||||
kernel_initializer = None,
|
||||
name = None):
|
||||
|
||||
conv_layer = tf.layers.conv2d(
|
||||
inputs = inputs,
|
||||
filters = filters,
|
||||
kernel_size = kernel_size,
|
||||
strides = strides,
|
||||
padding = padding,
|
||||
activation = activation,
|
||||
kernel_initializer = kernel_initializer,
|
||||
name = name)
|
||||
|
||||
return conv_layer
|
||||
|
||||
def residual1d_block(
|
||||
inputs,
|
||||
filters = 1024,
|
||||
kernel_size = 3,
|
||||
strides = 1,
|
||||
name_prefix = 'residule_block_'):
|
||||
|
||||
h1 = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv')
|
||||
h1_norm = instance_norm_layer(inputs = h1, activation_fn = None, name = name_prefix + 'h1_norm')
|
||||
h1_gates = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates')
|
||||
h1_norm_gates = instance_norm_layer(inputs = h1_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates')
|
||||
h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu')
|
||||
h2 = conv1d_layer(inputs = h1_glu, filters = filters // 2, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h2_conv')
|
||||
h2_norm = instance_norm_layer(inputs = h2, activation_fn = None, name = name_prefix + 'h2_norm')
|
||||
|
||||
h3 = inputs + h2_norm
|
||||
|
||||
return h3
|
||||
|
||||
def downsample1d_block(
|
||||
inputs,
|
||||
filters,
|
||||
kernel_size,
|
||||
strides,
|
||||
name_prefix = 'downsample1d_block_'):
|
||||
|
||||
h1 = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv')
|
||||
h1_norm = instance_norm_layer(inputs = h1, activation_fn = None, name = name_prefix + 'h1_norm')
|
||||
h1_gates = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates')
|
||||
h1_norm_gates = instance_norm_layer(inputs = h1_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates')
|
||||
h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu')
|
||||
|
||||
return h1_glu
|
||||
|
||||
def downsample2d_block(
|
||||
inputs,
|
||||
filters,
|
||||
kernel_size,
|
||||
strides,
|
||||
name_prefix = 'downsample2d_block_'):
|
||||
|
||||
h1 = conv2d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv')
|
||||
h1_norm = instance_norm_layer(inputs = h1, activation_fn = None, name = name_prefix + 'h1_norm')
|
||||
h1_gates = conv2d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates')
|
||||
h1_norm_gates = instance_norm_layer(inputs = h1_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates')
|
||||
h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu')
|
||||
|
||||
return h1_glu
|
||||
|
||||
def upsample1d_block(
|
||||
inputs,
|
||||
filters,
|
||||
kernel_size,
|
||||
strides,
|
||||
shuffle_size = 2,
|
||||
name_prefix = 'upsample1d_block_'):
|
||||
|
||||
h1 = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv')
|
||||
h1_shuffle = pixel_shuffler(inputs = h1, shuffle_size = shuffle_size, name = name_prefix + 'h1_shuffle')
|
||||
h1_norm = instance_norm_layer(inputs = h1_shuffle, activation_fn = None, name = name_prefix + 'h1_norm')
|
||||
|
||||
h1_gates = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates')
|
||||
h1_shuffle_gates = pixel_shuffler(inputs = h1_gates, shuffle_size = shuffle_size, name = name_prefix + 'h1_shuffle_gates')
|
||||
h1_norm_gates = instance_norm_layer(inputs = h1_shuffle_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates')
|
||||
|
||||
h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu')
|
||||
|
||||
return h1_glu
|
||||
|
||||
def pixel_shuffler(inputs, shuffle_size = 2, name = None):
|
||||
|
||||
n = tf.shape(inputs)[0]
|
||||
w = tf.shape(inputs)[1]
|
||||
c = inputs.get_shape().as_list()[2]
|
||||
|
||||
oc = c // shuffle_size
|
||||
ow = w * shuffle_size
|
||||
|
||||
outputs = tf.reshape(tensor = inputs, shape = [n, ow, oc], name = name)
|
||||
|
||||
return outputs
|
||||
|
||||
def generator_gatedcnn(inputs, reuse = False, scope_name = 'generator_gatedcnn'):
|
||||
|
||||
# inputs has shape [batch_size, num_features, time]
|
||||
# we need to convert it to [batch_size, time, num_features] for 1D convolution
|
||||
inputs = tf.transpose(inputs, perm = [0, 2, 1], name = 'input_transpose')
|
||||
|
||||
with tf.variable_scope(scope_name) as scope:
|
||||
# Discriminator would be reused in CycleGAN
|
||||
if reuse:
|
||||
scope.reuse_variables()
|
||||
else:
|
||||
assert scope.reuse is False
|
||||
|
||||
h1 = conv1d_layer(inputs = inputs, filters = 128, kernel_size = 15, strides = 1, activation = None, name = 'h1_conv')
|
||||
h1_gates = conv1d_layer(inputs = inputs, filters = 128, kernel_size = 15, strides = 1, activation = None, name = 'h1_conv_gates')
|
||||
h1_glu = gated_linear_layer(inputs = h1, gates = h1_gates, name = 'h1_glu')
|
||||
|
||||
# Downsample
|
||||
d1 = downsample1d_block(inputs = h1_glu, filters = 256, kernel_size = 5, strides = 2, name_prefix = 'downsample1d_block1_')
|
||||
d2 = downsample1d_block(inputs = d1, filters = 512, kernel_size = 5, strides = 2, name_prefix = 'downsample1d_block2_')
|
||||
|
||||
# Residual blocks
|
||||
r1 = residual1d_block(inputs = d2, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block1_')
|
||||
r2 = residual1d_block(inputs = r1, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block2_')
|
||||
r3 = residual1d_block(inputs = r2, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block3_')
|
||||
r4 = residual1d_block(inputs = r3, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block4_')
|
||||
r5 = residual1d_block(inputs = r4, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block5_')
|
||||
r6 = residual1d_block(inputs = r5, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block6_')
|
||||
|
||||
# Upsample
|
||||
u1 = upsample1d_block(inputs = r6, filters = 1024, kernel_size = 5, strides = 1, shuffle_size = 2, name_prefix = 'upsample1d_block1_')
|
||||
u2 = upsample1d_block(inputs = u1, filters = 512, kernel_size = 5, strides = 1, shuffle_size = 2, name_prefix = 'upsample1d_block2_')
|
||||
|
||||
# Output
|
||||
o1 = conv1d_layer(inputs = u2, filters = 24, kernel_size = 15, strides = 1, activation = None, name = 'o1_conv')
|
||||
o2 = tf.transpose(o1, perm = [0, 2, 1], name = 'output_transpose')
|
||||
|
||||
return o2
|
||||
|
||||
|
||||
def discriminator(inputs, reuse = False, scope_name = 'discriminator'):
|
||||
|
||||
# inputs has shape [batch_size, num_features, time]
|
||||
# we need to add channel for 2D convolution [batch_size, num_features, time, 1]
|
||||
inputs = tf.expand_dims(inputs, -1)
|
||||
|
||||
with tf.variable_scope(scope_name) as scope:
|
||||
# Discriminator would be reused in CycleGAN
|
||||
if reuse:
|
||||
scope.reuse_variables()
|
||||
else:
|
||||
assert scope.reuse is False
|
||||
|
||||
h1 = conv2d_layer(inputs = inputs, filters = 128, kernel_size = [3, 3], strides = [1, 2], activation = None, name = 'h1_conv')
|
||||
h1_gates = conv2d_layer(inputs = inputs, filters = 128, kernel_size = [3, 3], strides = [1, 2], activation = None, name = 'h1_conv_gates')
|
||||
h1_glu = gated_linear_layer(inputs = h1, gates = h1_gates, name = 'h1_glu')
|
||||
|
||||
# Downsample
|
||||
d1 = downsample2d_block(inputs = h1_glu, filters = 256, kernel_size = [3, 3], strides = [2, 2], name_prefix = 'downsample2d_block1_')
|
||||
d2 = downsample2d_block(inputs = d1, filters = 512, kernel_size = [3, 3], strides = [2, 2], name_prefix = 'downsample2d_block2_')
|
||||
d3 = downsample2d_block(inputs = d2, filters = 1024, kernel_size = [6, 3], strides = [1, 2], name_prefix = 'downsample2d_block3_')
|
||||
|
||||
# Output
|
||||
o1 = tf.layers.dense(inputs = d3, units = 1, activation = tf.nn.sigmoid)
|
||||
|
||||
return o1
|
||||
|
||||
@@ -1,238 +0,0 @@
|
||||
import librosa
|
||||
import numpy as np
|
||||
import os
|
||||
import pyworld
|
||||
|
||||
def load_wavs(wav_dir, sr):
|
||||
|
||||
wavs = list()
|
||||
for file in os.listdir(wav_dir):
|
||||
file_path = os.path.join(wav_dir, file)
|
||||
wav, _ = librosa.load(file_path, sr = sr, mono = True)
|
||||
#wav = wav.astype(np.float64)
|
||||
wavs.append(wav)
|
||||
|
||||
return wavs
|
||||
|
||||
def world_decompose(wav, fs, frame_period = 5.0):
|
||||
|
||||
# Decompose speech signal into f0, spectral envelope and aperiodicity using WORLD
|
||||
wav = wav.astype(np.float64)
|
||||
f0, timeaxis = pyworld.harvest(wav, fs, frame_period = frame_period, f0_floor = 71.0, f0_ceil = 800.0)
|
||||
sp = pyworld.cheaptrick(wav, f0, timeaxis, fs)
|
||||
ap = pyworld.d4c(wav, f0, timeaxis, fs)
|
||||
|
||||
return f0, timeaxis, sp, ap
|
||||
|
||||
def world_encode_spectral_envelop(sp, fs, dim = 24):
|
||||
|
||||
# Get Mel-cepstral coefficients (MCEPs)
|
||||
|
||||
#sp = sp.astype(np.float64)
|
||||
coded_sp = pyworld.code_spectral_envelope(sp, fs, dim)
|
||||
|
||||
return coded_sp
|
||||
|
||||
def world_decode_spectral_envelop(coded_sp, fs):
|
||||
|
||||
fftlen = pyworld.get_cheaptrick_fft_size(fs)
|
||||
#coded_sp = coded_sp.astype(np.float32)
|
||||
#coded_sp = np.ascontiguousarray(coded_sp)
|
||||
decoded_sp = pyworld.decode_spectral_envelope(coded_sp, fs, fftlen)
|
||||
|
||||
return decoded_sp
|
||||
|
||||
|
||||
def world_encode_data(wavs, fs, frame_period = 5.0, coded_dim = 24):
|
||||
|
||||
f0s = list()
|
||||
timeaxes = list()
|
||||
sps = list()
|
||||
aps = list()
|
||||
coded_sps = list()
|
||||
|
||||
for wav in wavs:
|
||||
f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = fs, frame_period = frame_period)
|
||||
coded_sp = world_encode_spectral_envelop(sp = sp, fs = fs, dim = coded_dim)
|
||||
f0s.append(f0)
|
||||
timeaxes.append(timeaxis)
|
||||
sps.append(sp)
|
||||
aps.append(ap)
|
||||
coded_sps.append(coded_sp)
|
||||
|
||||
return f0s, timeaxes, sps, aps, coded_sps
|
||||
|
||||
|
||||
def transpose_in_list(lst):
|
||||
|
||||
transposed_lst = list()
|
||||
for array in lst:
|
||||
transposed_lst.append(array.T)
|
||||
return transposed_lst
|
||||
|
||||
|
||||
def world_decode_data(coded_sps, fs):
|
||||
|
||||
decoded_sps = list()
|
||||
|
||||
for coded_sp in coded_sps:
|
||||
decoded_sp = world_decode_spectral_envelop(coded_sp, fs)
|
||||
decoded_sps.append(decoded_sp)
|
||||
|
||||
return decoded_sps
|
||||
|
||||
|
||||
def world_speech_synthesis(f0, decoded_sp, ap, fs, frame_period):
|
||||
|
||||
#decoded_sp = decoded_sp.astype(np.float64)
|
||||
wav = pyworld.synthesize(f0, decoded_sp, ap, fs, frame_period)
|
||||
# Librosa could not save wav if not doing so
|
||||
wav = wav.astype(np.float32)
|
||||
|
||||
return wav
|
||||
|
||||
|
||||
def world_synthesis_data(f0s, decoded_sps, aps, fs, frame_period):
|
||||
|
||||
wavs = list()
|
||||
|
||||
for f0, decoded_sp, ap in zip(f0s, decoded_sps, aps):
|
||||
wav = world_speech_synthesis(f0, decoded_sp, ap, fs, frame_period)
|
||||
wavs.append(wav)
|
||||
|
||||
return wavs
|
||||
|
||||
|
||||
def coded_sps_normalization_fit_transoform(coded_sps):
|
||||
|
||||
coded_sps_concatenated = np.concatenate(coded_sps, axis = 1)
|
||||
coded_sps_mean = np.mean(coded_sps_concatenated, axis = 1, keepdims = True)
|
||||
coded_sps_std = np.std(coded_sps_concatenated, axis = 1, keepdims = True)
|
||||
|
||||
coded_sps_normalized = list()
|
||||
for coded_sp in coded_sps:
|
||||
coded_sps_normalized.append((coded_sp - coded_sps_mean) / coded_sps_std)
|
||||
|
||||
return coded_sps_normalized, coded_sps_mean, coded_sps_std
|
||||
|
||||
def coded_sps_normalization_transoform(coded_sps, coded_sps_mean, coded_sps_std):
|
||||
|
||||
coded_sps_normalized = list()
|
||||
for coded_sp in coded_sps:
|
||||
coded_sps_normalized.append((coded_sp - coded_sps_mean) / coded_sps_std)
|
||||
|
||||
return coded_sps_normalized
|
||||
|
||||
def coded_sps_normalization_inverse_transoform(normalized_coded_sps, coded_sps_mean, coded_sps_std):
|
||||
|
||||
coded_sps = list()
|
||||
for normalized_coded_sp in normalized_coded_sps:
|
||||
coded_sps.append(normalized_coded_sp * coded_sps_std + coded_sps_mean)
|
||||
|
||||
return coded_sps
|
||||
|
||||
def coded_sp_padding(coded_sp, multiple = 4):
|
||||
|
||||
num_features = coded_sp.shape[0]
|
||||
num_frames = coded_sp.shape[1]
|
||||
num_frames_padded = int(np.ceil(num_frames / multiple)) * multiple
|
||||
num_frames_diff = num_frames_padded - num_frames
|
||||
num_pad_left = num_frames_diff // 2
|
||||
num_pad_right = num_frames_diff - num_pad_left
|
||||
coded_sp_padded = np.pad(coded_sp, ((0, 0), (num_pad_left, num_pad_right)), 'constant', constant_values = 0)
|
||||
|
||||
return coded_sp_padded
|
||||
|
||||
def wav_padding(wav, sr, frame_period, multiple = 4):
|
||||
|
||||
assert wav.ndim == 1
|
||||
num_frames = len(wav)
|
||||
num_frames_padded = int((np.ceil((np.floor(num_frames / (sr * frame_period / 1000)) + 1) / multiple + 1) * multiple - 1) * (sr * frame_period / 1000))
|
||||
num_frames_diff = num_frames_padded - num_frames
|
||||
num_pad_left = num_frames_diff // 2
|
||||
num_pad_right = num_frames_diff - num_pad_left
|
||||
wav_padded = np.pad(wav, (num_pad_left, num_pad_right), 'constant', constant_values = 0)
|
||||
|
||||
return wav_padded
|
||||
|
||||
|
||||
def logf0_statistics(f0s):
|
||||
|
||||
log_f0s_concatenated = np.ma.log(np.concatenate(f0s))
|
||||
log_f0s_mean = log_f0s_concatenated.mean()
|
||||
log_f0s_std = log_f0s_concatenated.std()
|
||||
|
||||
return log_f0s_mean, log_f0s_std
|
||||
|
||||
def pitch_conversion(f0, mean_log_src, std_log_src, mean_log_target, std_log_target):
|
||||
|
||||
# Logarithm Gaussian normalization for Pitch Conversions
|
||||
f0_converted = np.exp((np.log(f0) - mean_log_src) / std_log_src * std_log_target + mean_log_target)
|
||||
|
||||
return f0_converted
|
||||
|
||||
def wavs_to_specs(wavs, n_fft = 1024, hop_length = None):
|
||||
|
||||
stfts = list()
|
||||
for wav in wavs:
|
||||
stft = librosa.stft(wav, n_fft = n_fft, hop_length = hop_length)
|
||||
stfts.append(stft)
|
||||
|
||||
return stfts
|
||||
|
||||
|
||||
def wavs_to_mfccs(wavs, sr, n_fft = 1024, hop_length = None, n_mels = 128, n_mfcc = 24):
|
||||
|
||||
mfccs = list()
|
||||
for wav in wavs:
|
||||
mfcc = librosa.feature.mfcc(y = wav, sr = sr, n_fft = n_fft, hop_length = hop_length, n_mels = n_mels, n_mfcc = n_mfcc)
|
||||
mfccs.append(mfcc)
|
||||
|
||||
return mfccs
|
||||
|
||||
|
||||
def mfccs_normalization(mfccs):
|
||||
|
||||
mfccs_concatenated = np.concatenate(mfccs, axis = 1)
|
||||
mfccs_mean = np.mean(mfccs_concatenated, axis = 1, keepdims = True)
|
||||
mfccs_std = np.std(mfccs_concatenated, axis = 1, keepdims = True)
|
||||
|
||||
mfccs_normalized = list()
|
||||
for mfcc in mfccs:
|
||||
mfccs_normalized.append((mfcc - mfccs_mean) / mfccs_std)
|
||||
|
||||
return mfccs_normalized, mfccs_mean, mfccs_std
|
||||
|
||||
|
||||
def sample_train_data(dataset_A, dataset_B, n_frames = 128):
|
||||
|
||||
num_samples = min(len(dataset_A), len(dataset_B))
|
||||
train_data_A_idx = np.arange(len(dataset_A))
|
||||
train_data_B_idx = np.arange(len(dataset_B))
|
||||
np.random.shuffle(train_data_A_idx)
|
||||
np.random.shuffle(train_data_B_idx)
|
||||
train_data_A_idx_subset = train_data_A_idx[:num_samples]
|
||||
train_data_B_idx_subset = train_data_B_idx[:num_samples]
|
||||
|
||||
train_data_A = list()
|
||||
train_data_B = list()
|
||||
|
||||
for idx_A, idx_B in zip(train_data_A_idx_subset, train_data_B_idx_subset):
|
||||
data_A = dataset_A[idx_A]
|
||||
frames_A_total = data_A.shape[1]
|
||||
assert frames_A_total >= n_frames
|
||||
start_A = np.random.randint(frames_A_total - n_frames + 1)
|
||||
end_A = start_A + n_frames
|
||||
train_data_A.append(data_A[:,start_A:end_A])
|
||||
|
||||
data_B = dataset_B[idx_B]
|
||||
frames_B_total = data_B.shape[1]
|
||||
assert frames_B_total >= n_frames
|
||||
start_B = np.random.randint(frames_B_total - n_frames + 1)
|
||||
end_B = start_B + n_frames
|
||||
train_data_B.append(data_B[:,start_B:end_B])
|
||||
|
||||
train_data_A = np.array(train_data_A)
|
||||
train_data_B = np.array(train_data_B)
|
||||
|
||||
return train_data_A, train_data_B
|
||||
@@ -1,260 +0,0 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: protos/audio.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf import descriptor_pb2
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='protos/audio.proto',
|
||||
package='voder_audio',
|
||||
syntax='proto3',
|
||||
serialized_pb=_b('\n\x12protos/audio.proto\x12\x0bvoder_audio\"\xe6\x04\n\x12VocodeAudioRequest\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x1c\n\x14request_batch_number\x18\x02 \x01(\x03\x12\x43\n\rvocode_params\x18\n \x01(\x0b\x32,.voder_audio.VocodeAudioRequest.VocodeParams\x12\x13\n\x0bskip_vocode\x18\x04 \x01(\x08\x12\x1b\n\x13\x62uffer_size_minimum\x18\x06 \x01(\x05\x12\x1d\n\x15\x64iscard_vocoded_audio\x18\x08 \x01(\x08\x1a\x86\x03\n\x0cVocodeParams\x12\x1b\n\x13initial_sample_rate\x18\x01 \x01(\x05\x12\x1c\n\x14original_source_rate\x18\t \x01(\x05\x12!\n\x19original_source_save_file\x18\n \x01(\x08\x12\x1c\n\x14pre_convert_resample\x18\x02 \x01(\x08\x12!\n\x19pre_convert_resample_rate\x18\x03 \x01(\x05\x12&\n\x1epre_convert_resample_save_file\x18\x04 \x01(\x08\x12*\n\"model_hyperparameter_sampling_rate\x18\x05 \x01(\x05\x12\x17\n\x0fmodel_save_file\x18\x0b \x01(\x08\x12\x1d\n\x15post_convert_resample\x18\x06 \x01(\x08\x12\"\n\x1apost_convert_resample_rate\x18\x07 \x01(\x05\x12\'\n\x1fpost_convert_resample_save_file\x18\x08 \x01(\x08\"I\n\x13VocodeAudioResponse\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x1d\n\x15response_batch_number\x18\x02 \x01(\x03\x62\x06proto3')
|
||||
)
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
|
||||
|
||||
|
||||
_VOCODEAUDIOREQUEST_VOCODEPARAMS = _descriptor.Descriptor(
|
||||
name='VocodeParams',
|
||||
full_name='voder_audio.VocodeAudioRequest.VocodeParams',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='initial_sample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.initial_sample_rate', index=0,
|
||||
number=1, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='original_source_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.original_source_rate', index=1,
|
||||
number=9, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='original_source_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.original_source_save_file', index=2,
|
||||
number=10, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='pre_convert_resample', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample', index=3,
|
||||
number=2, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='pre_convert_resample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample_rate', index=4,
|
||||
number=3, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='pre_convert_resample_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample_save_file', index=5,
|
||||
number=4, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='model_hyperparameter_sampling_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.model_hyperparameter_sampling_rate', index=6,
|
||||
number=5, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='model_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.model_save_file', index=7,
|
||||
number=11, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='post_convert_resample', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample', index=8,
|
||||
number=6, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='post_convert_resample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample_rate', index=9,
|
||||
number=7, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='post_convert_resample_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample_save_file', index=10,
|
||||
number=8, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=260,
|
||||
serialized_end=650,
|
||||
)
|
||||
|
||||
_VOCODEAUDIOREQUEST = _descriptor.Descriptor(
|
||||
name='VocodeAudioRequest',
|
||||
full_name='voder_audio.VocodeAudioRequest',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='float_audio', full_name='voder_audio.VocodeAudioRequest.float_audio', index=0,
|
||||
number=1, type=2, cpp_type=6, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='request_batch_number', full_name='voder_audio.VocodeAudioRequest.request_batch_number', index=1,
|
||||
number=2, type=3, cpp_type=2, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='vocode_params', full_name='voder_audio.VocodeAudioRequest.vocode_params', index=2,
|
||||
number=10, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='skip_vocode', full_name='voder_audio.VocodeAudioRequest.skip_vocode', index=3,
|
||||
number=4, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='buffer_size_minimum', full_name='voder_audio.VocodeAudioRequest.buffer_size_minimum', index=4,
|
||||
number=6, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='discard_vocoded_audio', full_name='voder_audio.VocodeAudioRequest.discard_vocoded_audio', index=5,
|
||||
number=8, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[_VOCODEAUDIOREQUEST_VOCODEPARAMS, ],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=36,
|
||||
serialized_end=650,
|
||||
)
|
||||
|
||||
|
||||
_VOCODEAUDIORESPONSE = _descriptor.Descriptor(
|
||||
name='VocodeAudioResponse',
|
||||
full_name='voder_audio.VocodeAudioResponse',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='float_audio', full_name='voder_audio.VocodeAudioResponse.float_audio', index=0,
|
||||
number=1, type=2, cpp_type=6, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='response_batch_number', full_name='voder_audio.VocodeAudioResponse.response_batch_number', index=1,
|
||||
number=2, type=3, cpp_type=2, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=652,
|
||||
serialized_end=725,
|
||||
)
|
||||
|
||||
_VOCODEAUDIOREQUEST_VOCODEPARAMS.containing_type = _VOCODEAUDIOREQUEST
|
||||
_VOCODEAUDIOREQUEST.fields_by_name['vocode_params'].message_type = _VOCODEAUDIOREQUEST_VOCODEPARAMS
|
||||
DESCRIPTOR.message_types_by_name['VocodeAudioRequest'] = _VOCODEAUDIOREQUEST
|
||||
DESCRIPTOR.message_types_by_name['VocodeAudioResponse'] = _VOCODEAUDIORESPONSE
|
||||
|
||||
VocodeAudioRequest = _reflection.GeneratedProtocolMessageType('VocodeAudioRequest', (_message.Message,), dict(
|
||||
|
||||
VocodeParams = _reflection.GeneratedProtocolMessageType('VocodeParams', (_message.Message,), dict(
|
||||
DESCRIPTOR = _VOCODEAUDIOREQUEST_VOCODEPARAMS,
|
||||
__module__ = 'protos.audio_pb2'
|
||||
# @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioRequest.VocodeParams)
|
||||
))
|
||||
,
|
||||
DESCRIPTOR = _VOCODEAUDIOREQUEST,
|
||||
__module__ = 'protos.audio_pb2'
|
||||
# @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioRequest)
|
||||
))
|
||||
_sym_db.RegisterMessage(VocodeAudioRequest)
|
||||
_sym_db.RegisterMessage(VocodeAudioRequest.VocodeParams)
|
||||
|
||||
VocodeAudioResponse = _reflection.GeneratedProtocolMessageType('VocodeAudioResponse', (_message.Message,), dict(
|
||||
DESCRIPTOR = _VOCODEAUDIORESPONSE,
|
||||
__module__ = 'protos.audio_pb2'
|
||||
# @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioResponse)
|
||||
))
|
||||
_sym_db.RegisterMessage(VocodeAudioResponse)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -1,36 +0,0 @@
|
||||
absl-py==0.7.1
|
||||
astor==0.8.0
|
||||
audioread==2.1.8
|
||||
cffi==1.12.3
|
||||
Cython==0.29.12
|
||||
decorator==4.4.0
|
||||
falcon==2.0.0
|
||||
falcon-multipart==0.2.0
|
||||
gast==0.2.2
|
||||
google-pasta==0.1.7
|
||||
grpcio==1.22.0
|
||||
h5py==2.9.0
|
||||
joblib==0.13.2
|
||||
Keras-Applications==1.0.8
|
||||
Keras-Preprocessing==1.1.0
|
||||
librosa==0.7.0
|
||||
llvmlite==0.29.0
|
||||
Markdown==3.1.1
|
||||
numba==0.45.0
|
||||
numpy==1.16.4
|
||||
protobuf==3.9.0
|
||||
pycparser==2.19
|
||||
pyworld==0.2.8
|
||||
pyzmq==18.1.0
|
||||
resampy==0.2.1
|
||||
scikit-learn==0.21.2
|
||||
scipy==1.3.0
|
||||
six==1.12.0
|
||||
SoundFile==0.10.2
|
||||
tensorboard==1.14.0
|
||||
tensorflow-estimator==1.14.0
|
||||
tensorflow-gpu==1.14.0
|
||||
termcolor==1.1.0
|
||||
Werkzeug==0.15.5
|
||||
wrapt==1.11.2
|
||||
zmq==0.0.0
|
||||
@@ -1,357 +0,0 @@
|
||||
(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.Recorder = f()}})(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({1:[function(require,module,exports){
|
||||
"use strict";
|
||||
|
||||
module.exports = require("./recorder").Recorder;
|
||||
|
||||
},{"./recorder":2}],2:[function(require,module,exports){
|
||||
'use strict';
|
||||
|
||||
var _createClass = (function () {
|
||||
function defineProperties(target, props) {
|
||||
for (var i = 0; i < props.length; i++) {
|
||||
var descriptor = props[i];descriptor.enumerable = descriptor.enumerable || false;descriptor.configurable = true;if ("value" in descriptor) descriptor.writable = true;Object.defineProperty(target, descriptor.key, descriptor);
|
||||
}
|
||||
}return function (Constructor, protoProps, staticProps) {
|
||||
if (protoProps) defineProperties(Constructor.prototype, protoProps);if (staticProps) defineProperties(Constructor, staticProps);return Constructor;
|
||||
};
|
||||
})();
|
||||
|
||||
Object.defineProperty(exports, "__esModule", {
|
||||
value: true
|
||||
});
|
||||
exports.Recorder = undefined;
|
||||
|
||||
var _inlineWorker = require('inline-worker');
|
||||
|
||||
var _inlineWorker2 = _interopRequireDefault(_inlineWorker);
|
||||
|
||||
function _interopRequireDefault(obj) {
|
||||
return obj && obj.__esModule ? obj : { default: obj };
|
||||
}
|
||||
|
||||
function _classCallCheck(instance, Constructor) {
|
||||
if (!(instance instanceof Constructor)) {
|
||||
throw new TypeError("Cannot call a class as a function");
|
||||
}
|
||||
}
|
||||
|
||||
var Recorder = exports.Recorder = (function () {
|
||||
function Recorder(source, cfg) {
|
||||
var _this = this;
|
||||
|
||||
_classCallCheck(this, Recorder);
|
||||
|
||||
this.config = {
|
||||
bufferLen: 4096,
|
||||
numChannels: 2,
|
||||
mimeType: 'audio/wav'
|
||||
};
|
||||
this.recording = false;
|
||||
this.callbacks = {
|
||||
getBuffer: [],
|
||||
exportWAV: []
|
||||
};
|
||||
|
||||
Object.assign(this.config, cfg);
|
||||
this.context = source.context;
|
||||
this.node = (this.context.createScriptProcessor || this.context.createJavaScriptNode).call(this.context, this.config.bufferLen, this.config.numChannels, this.config.numChannels);
|
||||
|
||||
this.node.onaudioprocess = function (e) {
|
||||
if (!_this.recording) return;
|
||||
|
||||
var buffer = [];
|
||||
for (var channel = 0; channel < _this.config.numChannels; channel++) {
|
||||
buffer.push(e.inputBuffer.getChannelData(channel));
|
||||
}
|
||||
_this.worker.postMessage({
|
||||
command: 'record',
|
||||
buffer: buffer
|
||||
});
|
||||
};
|
||||
|
||||
source.connect(this.node);
|
||||
this.node.connect(this.context.destination); //this should not be necessary
|
||||
|
||||
var self = {};
|
||||
this.worker = new _inlineWorker2.default(function () {
|
||||
var recLength = 0,
|
||||
recBuffers = [],
|
||||
sampleRate = undefined,
|
||||
numChannels = undefined;
|
||||
|
||||
self.onmessage = function (e) {
|
||||
switch (e.data.command) {
|
||||
case 'init':
|
||||
init(e.data.config);
|
||||
break;
|
||||
case 'record':
|
||||
record(e.data.buffer);
|
||||
break;
|
||||
case 'exportWAV':
|
||||
exportWAV(e.data.type);
|
||||
break;
|
||||
case 'getBuffer':
|
||||
getBuffer();
|
||||
break;
|
||||
case 'clear':
|
||||
clear();
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
function init(config) {
|
||||
sampleRate = config.sampleRate;
|
||||
numChannels = config.numChannels;
|
||||
initBuffers();
|
||||
}
|
||||
|
||||
function record(inputBuffer) {
|
||||
for (var channel = 0; channel < numChannels; channel++) {
|
||||
recBuffers[channel].push(inputBuffer[channel]);
|
||||
}
|
||||
recLength += inputBuffer[0].length;
|
||||
}
|
||||
|
||||
function exportWAV(type) {
|
||||
var buffers = [];
|
||||
for (var channel = 0; channel < numChannels; channel++) {
|
||||
buffers.push(mergeBuffers(recBuffers[channel], recLength));
|
||||
}
|
||||
var interleaved = undefined;
|
||||
if (numChannels === 2) {
|
||||
interleaved = interleave(buffers[0], buffers[1]);
|
||||
} else {
|
||||
interleaved = buffers[0];
|
||||
}
|
||||
var dataview = encodeWAV(interleaved);
|
||||
var audioBlob = new Blob([dataview], { type: type });
|
||||
|
||||
self.postMessage({ command: 'exportWAV', data: audioBlob });
|
||||
}
|
||||
|
||||
function getBuffer() {
|
||||
var buffers = [];
|
||||
for (var channel = 0; channel < numChannels; channel++) {
|
||||
buffers.push(mergeBuffers(recBuffers[channel], recLength));
|
||||
}
|
||||
self.postMessage({ command: 'getBuffer', data: buffers });
|
||||
}
|
||||
|
||||
function clear() {
|
||||
recLength = 0;
|
||||
recBuffers = [];
|
||||
initBuffers();
|
||||
}
|
||||
|
||||
function initBuffers() {
|
||||
for (var channel = 0; channel < numChannels; channel++) {
|
||||
recBuffers[channel] = [];
|
||||
}
|
||||
}
|
||||
|
||||
function mergeBuffers(recBuffers, recLength) {
|
||||
var result = new Float32Array(recLength);
|
||||
var offset = 0;
|
||||
for (var i = 0; i < recBuffers.length; i++) {
|
||||
result.set(recBuffers[i], offset);
|
||||
offset += recBuffers[i].length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function interleave(inputL, inputR) {
|
||||
var length = inputL.length + inputR.length;
|
||||
var result = new Float32Array(length);
|
||||
|
||||
var index = 0,
|
||||
inputIndex = 0;
|
||||
|
||||
while (index < length) {
|
||||
result[index++] = inputL[inputIndex];
|
||||
result[index++] = inputR[inputIndex];
|
||||
inputIndex++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function floatTo16BitPCM(output, offset, input) {
|
||||
for (var i = 0; i < input.length; i++, offset += 2) {
|
||||
var s = Math.max(-1, Math.min(1, input[i]));
|
||||
output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
|
||||
}
|
||||
}
|
||||
|
||||
function writeString(view, offset, string) {
|
||||
for (var i = 0; i < string.length; i++) {
|
||||
view.setUint8(offset + i, string.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
|
||||
function encodeWAV(samples) {
|
||||
var buffer = new ArrayBuffer(44 + samples.length * 2);
|
||||
var view = new DataView(buffer);
|
||||
|
||||
/* RIFF identifier */
|
||||
writeString(view, 0, 'RIFF');
|
||||
/* RIFF chunk length */
|
||||
view.setUint32(4, 36 + samples.length * 2, true);
|
||||
/* RIFF type */
|
||||
writeString(view, 8, 'WAVE');
|
||||
/* format chunk identifier */
|
||||
writeString(view, 12, 'fmt ');
|
||||
/* format chunk length */
|
||||
view.setUint32(16, 16, true);
|
||||
/* sample format (raw) */
|
||||
view.setUint16(20, 1, true);
|
||||
/* channel count */
|
||||
view.setUint16(22, numChannels, true);
|
||||
/* sample rate */
|
||||
view.setUint32(24, sampleRate, true);
|
||||
/* byte rate (sample rate * block align) */
|
||||
view.setUint32(28, sampleRate * 4, true);
|
||||
/* block align (channel count * bytes per sample) */
|
||||
view.setUint16(32, numChannels * 2, true);
|
||||
/* bits per sample */
|
||||
view.setUint16(34, 16, true);
|
||||
/* data chunk identifier */
|
||||
writeString(view, 36, 'data');
|
||||
/* data chunk length */
|
||||
view.setUint32(40, samples.length * 2, true);
|
||||
|
||||
floatTo16BitPCM(view, 44, samples);
|
||||
|
||||
return view;
|
||||
}
|
||||
}, self);
|
||||
|
||||
this.worker.postMessage({
|
||||
command: 'init',
|
||||
config: {
|
||||
sampleRate: this.context.sampleRate,
|
||||
numChannels: this.config.numChannels
|
||||
}
|
||||
});
|
||||
|
||||
this.worker.onmessage = function (e) {
|
||||
var cb = _this.callbacks[e.data.command].pop();
|
||||
if (typeof cb == 'function') {
|
||||
cb(e.data.data);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
_createClass(Recorder, [{
|
||||
key: 'record',
|
||||
value: function record() {
|
||||
this.recording = true;
|
||||
}
|
||||
}, {
|
||||
key: 'stop',
|
||||
value: function stop() {
|
||||
this.recording = false;
|
||||
}
|
||||
}, {
|
||||
key: 'clear',
|
||||
value: function clear() {
|
||||
this.worker.postMessage({ command: 'clear' });
|
||||
}
|
||||
}, {
|
||||
key: 'getBuffer',
|
||||
value: function getBuffer(cb) {
|
||||
cb = cb || this.config.callback;
|
||||
if (!cb) throw new Error('Callback not set');
|
||||
|
||||
this.callbacks.getBuffer.push(cb);
|
||||
|
||||
this.worker.postMessage({ command: 'getBuffer' });
|
||||
}
|
||||
}, {
|
||||
key: 'exportWAV',
|
||||
value: function exportWAV(cb, mimeType) {
|
||||
mimeType = mimeType || this.config.mimeType;
|
||||
cb = cb || this.config.callback;
|
||||
if (!cb) throw new Error('Callback not set');
|
||||
|
||||
this.callbacks.exportWAV.push(cb);
|
||||
|
||||
this.worker.postMessage({
|
||||
command: 'exportWAV',
|
||||
type: mimeType
|
||||
});
|
||||
}
|
||||
}], [{
|
||||
key: 'forceDownload',
|
||||
value: function forceDownload(blob, filename) {
|
||||
var url = (window.URL || window.webkitURL).createObjectURL(blob);
|
||||
var link = window.document.createElement('a');
|
||||
link.href = url;
|
||||
link.download = filename || 'output.wav';
|
||||
var click = document.createEvent("Event");
|
||||
click.initEvent("click", true, true);
|
||||
link.dispatchEvent(click);
|
||||
}
|
||||
}]);
|
||||
|
||||
return Recorder;
|
||||
})();
|
||||
|
||||
exports.default = Recorder;
|
||||
|
||||
},{"inline-worker":3}],3:[function(require,module,exports){
|
||||
"use strict";
|
||||
|
||||
module.exports = require("./inline-worker");
|
||||
},{"./inline-worker":4}],4:[function(require,module,exports){
|
||||
(function (global){
|
||||
"use strict";
|
||||
|
||||
var _createClass = (function () { function defineProperties(target, props) { for (var key in props) { var prop = props[key]; prop.configurable = true; if (prop.value) prop.writable = true; } Object.defineProperties(target, props); } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; })();
|
||||
|
||||
var _classCallCheck = function (instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } };
|
||||
|
||||
var WORKER_ENABLED = !!(global === global.window && global.URL && global.Blob && global.Worker);
|
||||
|
||||
var InlineWorker = (function () {
|
||||
function InlineWorker(func, self) {
|
||||
var _this = this;
|
||||
|
||||
_classCallCheck(this, InlineWorker);
|
||||
|
||||
if (WORKER_ENABLED) {
|
||||
var functionBody = func.toString().trim().match(/^function\s*\w*\s*\([\w\s,]*\)\s*{([\w\W]*?)}$/)[1];
|
||||
var url = global.URL.createObjectURL(new global.Blob([functionBody], { type: "text/javascript" }));
|
||||
|
||||
return new global.Worker(url);
|
||||
}
|
||||
|
||||
this.self = self;
|
||||
this.self.postMessage = function (data) {
|
||||
setTimeout(function () {
|
||||
_this.onmessage({ data: data });
|
||||
}, 0);
|
||||
};
|
||||
|
||||
setTimeout(function () {
|
||||
func.call(self);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
_createClass(InlineWorker, {
|
||||
postMessage: {
|
||||
value: function postMessage(data) {
|
||||
var _this = this;
|
||||
|
||||
setTimeout(function () {
|
||||
_this.self.onmessage({ data: data });
|
||||
}, 0);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return InlineWorker;
|
||||
})();
|
||||
|
||||
module.exports = InlineWorker;
|
||||
}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {})
|
||||
},{}]},{},[1])(1)
|
||||
});
|
||||
@@ -1,264 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Starts a demo HTTP server to capture and transform audio
|
||||
as a live demonstration of the trained model.
|
||||
|
||||
Brandon Thomas 2019-07-29 <bt@brand.io> <echelon@gmail.com>
|
||||
"""
|
||||
|
||||
# noinspection PyInterpreter
|
||||
import argparse
|
||||
import io
|
||||
import librosa
|
||||
import numpy as np
|
||||
import os
|
||||
import pathlib
|
||||
import scipy
|
||||
import soundfile
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import tensorflow as tf
|
||||
import zmq
|
||||
|
||||
from model import CycleGAN
|
||||
from preprocess import *
|
||||
from protos.audio_pb2 import VocodeAudioRequest
|
||||
from protos.audio_pb2 import VocodeAudioResponse
|
||||
|
||||
print("TensorFlow version: {}".format(tf.version.VERSION))
|
||||
|
||||
|
||||
class Converter():
|
||||
def __init__(self, model_dir, model_name):
|
||||
self.num_features = 24
|
||||
self.frame_period = 5.0
|
||||
|
||||
self.model = CycleGAN(num_features = self.num_features, mode = 'test')
|
||||
|
||||
self.model.load(filepath = os.path.join(model_dir, model_name))
|
||||
|
||||
self.mcep_normalization_params = np.load(os.path.join(model_dir, 'mcep_normalization.npz'))
|
||||
self.mcep_mean_A = self.mcep_normalization_params['mean_A']
|
||||
self.mcep_std_A = self.mcep_normalization_params['std_A']
|
||||
self.mcep_mean_B = self.mcep_normalization_params['mean_B']
|
||||
self.mcep_std_B = self.mcep_normalization_params['std_B']
|
||||
|
||||
self.logf0s_normalization_params = np.load(os.path.join(model_dir,
|
||||
'logf0s_normalization.npz'))
|
||||
self.logf0s_mean_A = self.logf0s_normalization_params['mean_A']
|
||||
self.logf0s_std_A = self.logf0s_normalization_params['std_A']
|
||||
self.logf0s_mean_B = self.logf0s_normalization_params['mean_B']
|
||||
self.logf0s_std_B = self.logf0s_normalization_params['std_B']
|
||||
|
||||
def convert_partial(self, wav, conversion_direction='A2B', model_sampling_rate=16000):
|
||||
wav = wav_padding(wav = wav,
|
||||
sr = model_sampling_rate,
|
||||
frame_period = self.frame_period,
|
||||
multiple = 4)
|
||||
|
||||
#librosa.output.write_wav('wav_padding.wav', wav, model_sampling_rate)
|
||||
|
||||
f0, timeaxis, sp, ap = world_decompose(wav = wav,
|
||||
fs = model_sampling_rate,
|
||||
frame_period = self.frame_period)
|
||||
coded_sp = world_encode_spectral_envelop(sp = sp,
|
||||
fs = model_sampling_rate,
|
||||
dim = self.num_features)
|
||||
coded_sp_transposed = coded_sp.T
|
||||
|
||||
f0_converted = pitch_conversion(f0 = f0,
|
||||
mean_log_src = self.logf0s_mean_A,
|
||||
std_log_src = self.logf0s_std_A,
|
||||
mean_log_target = self.logf0s_mean_B,
|
||||
std_log_target = self.logf0s_std_B)
|
||||
coded_sp_norm = (coded_sp_transposed - self.mcep_mean_A) / self.mcep_std_A
|
||||
|
||||
coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]),
|
||||
direction = conversion_direction)[0]
|
||||
coded_sp_converted = coded_sp_converted_norm * self.mcep_std_B + self.mcep_mean_B
|
||||
|
||||
coded_sp_converted = coded_sp_converted.T
|
||||
coded_sp_converted = np.ascontiguousarray(coded_sp_converted)
|
||||
decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted,
|
||||
fs = model_sampling_rate)
|
||||
wav_transformed = world_speech_synthesis(f0 = f0_converted,
|
||||
decoded_sp = decoded_sp_converted,
|
||||
ap = ap,
|
||||
fs = model_sampling_rate,
|
||||
frame_period = self.frame_period)
|
||||
|
||||
# For debugging model output, uncomment the following line:
|
||||
#librosa.output.write_wav('model_output.wav', wav_transformed, model_sampling_rate)
|
||||
|
||||
# TODO: Perhaps ditch this. It's probably unnecessary work.
|
||||
#upsampled = librosa.resample(wav_transformed, model_sampling_rate, 48000)
|
||||
#pcm_data = upsampled.astype(np.float64)
|
||||
#stereo_pcm_data = np.tile(pcm_data, (2,1)).T
|
||||
#return stereo_pcm_data.astype(np.float32)
|
||||
#librosa.output.write_wav('model_output.wav', pcm_data, 48000)
|
||||
|
||||
#return wav
|
||||
return wav_transformed
|
||||
|
||||
#def convert(self, wav, conversion_direction='A2B'):
|
||||
# pcm_data = self.convert_partial(wav, conversion_direction=conversion_direction)
|
||||
# buf = io.BytesIO()
|
||||
# # pcm_data: A 1-D or 2-D numpy array of either integer or float data-type.
|
||||
# # To write multiple-channels, use a 2-D array of shape (Nsamples, Nchannels).
|
||||
# scipy.io.wavfile.write(buf, 48000, pcm_data)
|
||||
# return buf
|
||||
|
||||
# Set up model
|
||||
# This should live long in memory, so we do it up front.
|
||||
model_dir_default = './model/sf1_tm1'
|
||||
model_name_default = 'sf1_tm1.ckpt'
|
||||
|
||||
# TODO: UNCOMMENT
|
||||
converter = Converter(model_dir_default, model_name_default)
|
||||
|
||||
TEMP_DIR = tempfile.TemporaryDirectory(prefix='queue_audio')
|
||||
|
||||
def temp_file_name(suffix='.wav'):
|
||||
# NB: Not actually using the tempfile. Just the random name.
|
||||
temp_file = tempfile.NamedTemporaryFile(suffix=suffix)
|
||||
name = os.path.basename(temp_file.name)
|
||||
return os.path.join(TEMP_DIR.name, name)
|
||||
|
||||
def convert(audio,
|
||||
vocode_params=None,
|
||||
request_batch_number=0,
|
||||
skip_vocode=False,
|
||||
save_files=False,
|
||||
discard_vocoded_audio=False):
|
||||
#audio = np.array(audio, dtype=np.int16)
|
||||
#data, samplerate = soundfile.read(audio)
|
||||
#print('samplerate', samplerate)
|
||||
"""
|
||||
samplerate 44100
|
||||
data.shape (77824, 2)
|
||||
data.dtype float64
|
||||
mono [0.0050354 0.00518799 0.0050354 ... 0.11651611 0.11935425 0.1164856 ]
|
||||
mono.shape (77824,)
|
||||
mono.dtype float64
|
||||
downsampled [0.00329925 0.00577342 0.00474898 ... 0.0851728 0.1166483 0. ]
|
||||
downsampled.shape (28236,)
|
||||
downsampled.dtype float64
|
||||
|
||||
1) Data should be float64
|
||||
2) Output is BYTES!! Not floats.
|
||||
3) Result is still somehow mono!?
|
||||
--------------------------------------------------
|
||||
|
||||
demo_server.py
|
||||
|
||||
capture sample rate = 44100
|
||||
downsample for algorithm = 16000
|
||||
upsample after done = 48000
|
||||
|
||||
"""
|
||||
|
||||
audio = np.array(audio, dtype=np.float64)
|
||||
print('audio.shape', audio.shape)
|
||||
print('audio.type', audio.dtype)
|
||||
|
||||
#source_rate = 88000 # Experimentally determined for Rust library 'CPAL'
|
||||
#source_rate = 44100
|
||||
|
||||
if vocode_params.original_source_save_file:
|
||||
#filename = temp_file_name('.wav')
|
||||
filename = 'debug/{}_original_source.wav'.format(request_batch_number)
|
||||
print('----- Original wav file out: {}'.format(filename))
|
||||
scipy.io.wavfile.write(filename, vocode_params.original_source_rate, audio)
|
||||
|
||||
# NB: Convert the input stereo signal into mono.
|
||||
# In the future the frontend should be responsible for sampling details.
|
||||
#audio = audio[:, 0]
|
||||
|
||||
if vocode_params.pre_convert_resample:
|
||||
print("Resampling audio from {} Hz to {} Hz".format(vocode_params.original_source_rate,
|
||||
vocode_params.pre_convert_resample_rate))
|
||||
audio = librosa.resample(audio, vocode_params.original_source_rate, vocode_params.pre_convert_resample_rate)
|
||||
print('resampled_audio.shape', audio.shape)
|
||||
print('resampled_audio.type', audio.dtype)
|
||||
if vocode_params.pre_convert_resample_save_file:
|
||||
#filename = temp_file_name('.wav')
|
||||
filename = 'debug/{}_pre_convert_resample.wav'.format(request_batch_number)
|
||||
print('----- Pre-downsampled file out: {}'.format(filename))
|
||||
scipy.io.wavfile.write(filename, vocode_params.pre_convert_resample_rate, audio)
|
||||
|
||||
if skip_vocode:
|
||||
return audio
|
||||
|
||||
results = converter.convert_partial(audio,
|
||||
conversion_direction='A2B',
|
||||
model_sampling_rate=vocode_params.model_hyperparameter_sampling_rate)
|
||||
|
||||
if vocode_params.model_save_file:
|
||||
#filename = temp_file_name('.wav')
|
||||
filename = 'debug/{}_model_output.wav'.format(request_batch_number)
|
||||
print('----- Model file out: {}'.format(filename))
|
||||
scipy.io.wavfile.write(filename, vocode_params.model_hyperparameter_sampling_rate, results)
|
||||
|
||||
print('results.type', type(results))
|
||||
print('results.len', len(results))
|
||||
|
||||
if vocode_params.post_convert_resample:
|
||||
print("Resampling output audio from {} Hz to {} Hz".format(vocode_params.model_hyperparameter_sampling_rate,
|
||||
vocode_params.post_convert_resample_rate))
|
||||
#consume_rate = 68000 # Experimentally determined for Rust lib 'CPAL'
|
||||
results = librosa.resample(results,
|
||||
vocode_params.model_hyperparameter_sampling_rate,
|
||||
vocode_params.post_convert_resample_rate)
|
||||
if vocode_params.post_convert_resample_save_file:
|
||||
#filename = temp_file_name('.wav')
|
||||
filename = 'debug/{}_post_convert_resample.wav'.format(request_batch_number)
|
||||
print('----- Upsampled (transformed) file out: {}'.format(filename))
|
||||
scipy.io.wavfile.write(filename, vocode_params.post_convert_resample_rate, results)
|
||||
|
||||
if discard_vocoded_audio:
|
||||
return audio
|
||||
else:
|
||||
return results
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--port', type=int, default=5555)
|
||||
args = parser.parse_args()
|
||||
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.REP)
|
||||
socket.bind("tcp://*:{}".format(args.port))
|
||||
print('Running server...')
|
||||
|
||||
queue = []
|
||||
while True:
|
||||
# Wait for next request from client
|
||||
message = socket.recv()
|
||||
|
||||
vocode_request = VocodeAudioRequest.FromString(message)
|
||||
vocode_params = vocode_request.vocode_params
|
||||
|
||||
queue.extend(vocode_request.float_audio)
|
||||
|
||||
if len(queue) >= vocode_request.buffer_size_minimum:
|
||||
#results = queue[:]
|
||||
results = convert(queue,
|
||||
vocode_params=vocode_params,
|
||||
request_batch_number=vocode_request.request_batch_number,
|
||||
skip_vocode=vocode_request.skip_vocode,
|
||||
discard_vocoded_audio=vocode_request.discard_vocoded_audio)
|
||||
queue = []
|
||||
|
||||
vocode_response = VocodeAudioResponse()
|
||||
vocode_response.float_audio[:] = results
|
||||
|
||||
socket.send(vocode_response.SerializeToString())
|
||||
else:
|
||||
# Must send reply back to client
|
||||
socket.send(b"OK")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
import tensorflow as tf
|
||||
import os
|
||||
import random
|
||||
import numpy as np
|
||||
|
||||
def l1_loss(y, y_hat):
|
||||
|
||||
return tf.reduce_mean(tf.abs(y - y_hat))
|
||||
|
||||
def l2_loss(y, y_hat):
|
||||
|
||||
return tf.reduce_mean(tf.square(y - y_hat))
|
||||
|
||||
def cross_entropy_loss(logits, labels):
|
||||
return tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = logits, labels = labels))
|
||||
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
syntax = "proto3";
|
||||
package voder_audio;
|
||||
|
||||
message VocodeAudioRequest {
|
||||
repeated float float_audio = 1;
|
||||
|
||||
// An autoincrement sent with each request.
|
||||
int64 request_batch_number = 2;
|
||||
|
||||
message VocodeParams {
|
||||
// The initial sample rate coming from the microphone.
|
||||
int32 initial_sample_rate = 1;
|
||||
|
||||
// Original sample
|
||||
int32 original_source_rate = 9;
|
||||
bool original_source_save_file = 10;
|
||||
|
||||
// Resample before passing to the algorithm?
|
||||
bool pre_convert_resample = 2;
|
||||
int32 pre_convert_resample_rate = 3;
|
||||
bool pre_convert_resample_save_file = 4;
|
||||
|
||||
// The hard-coded hyperparameter of the model.
|
||||
// Probably don't want to change from 16000.
|
||||
int32 model_hyperparameter_sampling_rate = 5;
|
||||
bool model_save_file = 11;
|
||||
|
||||
// Resample after passing to the algorithm?
|
||||
bool post_convert_resample = 6;
|
||||
int32 post_convert_resample_rate = 7;
|
||||
bool post_convert_resample_save_file = 8;
|
||||
}
|
||||
|
||||
VocodeParams vocode_params = 10;
|
||||
|
||||
// We'll resend these params for every batch, but will only use
|
||||
// the last batch's settings. Kind of a dumb API, but it'll work.
|
||||
//int32 sample_rate = 2;
|
||||
|
||||
//bool skip_resample = 3;
|
||||
bool skip_vocode = 4;
|
||||
|
||||
// Debugging
|
||||
//bool save_files = 5;
|
||||
|
||||
// How big we let the buffer grow before running 'convert'.
|
||||
int32 buffer_size_minimum = 6;
|
||||
|
||||
// Requested output rate
|
||||
//int32 output_rate = 7;
|
||||
|
||||
// Discard the vocoded audio and return the original
|
||||
bool discard_vocoded_audio = 8;
|
||||
|
||||
// Rate param for the model
|
||||
//int32 model_sampling_rate = 9;
|
||||
}
|
||||
|
||||
message VocodeAudioResponse {
|
||||
repeated float float_audio = 1;
|
||||
|
||||
// A sidecar-side autoincrement sent with each request.
|
||||
int64 response_batch_number = 2;
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
use cpal::traits::{DeviceTrait, EventLoopTrait, HostTrait};
|
||||
use cpal;
|
||||
use audio_queue::AudioQueue;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// CPAL Audio Device
|
||||
/// Access audio input and output streams.
|
||||
pub struct AudioHardware {
|
||||
host: cpal::Host,
|
||||
event_loop: cpal::EventLoop,
|
||||
microphone_queue: Arc<AudioQueue>,
|
||||
speaker_queue: Arc<AudioQueue>,
|
||||
}
|
||||
|
||||
impl AudioHardware {
|
||||
pub fn new(microphone_queue: Arc<AudioQueue>, speaker_queue: Arc<AudioQueue>) -> Result<Self, failure::Error> {
|
||||
let host = cpal::default_host();
|
||||
let event_loop = host.event_loop();
|
||||
|
||||
let input_device = host.default_input_device().expect("failed to get default input device");
|
||||
let output_device = host.default_output_device().expect("failed to get default output device");
|
||||
|
||||
println!("Using default input device: \"{}\"", input_device.name()?);
|
||||
println!("Using default output device: \"{}\"", output_device.name()?);
|
||||
|
||||
// We'll try and use the same format between streams to keep it simple
|
||||
let mut format = input_device.default_input_format()?;
|
||||
format.data_type = cpal::SampleFormat::F32;
|
||||
|
||||
// Build streams.
|
||||
println!("Attempting to build both streams with `{:?}`.", format);
|
||||
let input_stream_id = event_loop.build_input_stream(&input_device, &format)?;
|
||||
let output_stream_id = event_loop.build_output_stream(&output_device, &format)?;
|
||||
println!("Successfully built streams.");
|
||||
|
||||
Ok(AudioHardware {
|
||||
host,
|
||||
event_loop,
|
||||
microphone_queue,
|
||||
speaker_queue,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn run(&mut self) {
|
||||
self.event_loop.run(|id, result| {
|
||||
let data = match result {
|
||||
Ok(data) => data,
|
||||
Err(err) => {
|
||||
eprintln!("An error occurred on stream {:?}: {}", id, err);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
match data {
|
||||
cpal::StreamData::Input { buffer: cpal::UnknownTypeInputBuffer::F32(buffer) } => {
|
||||
//assert_eq!(id, input_stream_id);
|
||||
let mut output_fell_behind = false;
|
||||
for &sample in buffer.iter() {
|
||||
self.microphone_queue.push_back(sample);
|
||||
}
|
||||
},
|
||||
cpal::StreamData::Output { buffer: cpal::UnknownTypeOutputBuffer::F32(mut buffer) } => {
|
||||
//println!("Audio out buffer len: {}", self.speaker_queue.len());
|
||||
let request_size = buffer.len();
|
||||
let mut drained = self.speaker_queue.drain_size((request_size));
|
||||
match drained {
|
||||
None => {
|
||||
for sample in buffer.iter_mut() {
|
||||
*sample = 0.0;
|
||||
}
|
||||
},
|
||||
Some(mut drained) => {
|
||||
for (i, sample) in buffer.iter_mut().enumerate() {
|
||||
*sample = drained.get(i).copied().unwrap();
|
||||
//*sample = 0.0;
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
_ => panic!("We're expecting f32 data."),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
use zmq;
|
||||
use zmq::{Error, Socket};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::{RwLock, PoisonError};
|
||||
|
||||
pub struct AudioQueue {
|
||||
queue: RwLock<VecDeque<f32>>,
|
||||
}
|
||||
|
||||
impl AudioQueue {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
queue: RwLock::new(VecDeque::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push_back(&self, val: f32) {
|
||||
match self.queue.write() {
|
||||
Ok(mut queue) => {
|
||||
queue.push_back(val);
|
||||
},
|
||||
Err(_) => {
|
||||
unreachable!("This shouldn't happen (push_back)");
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extend(&self, buf: Vec<f32>) {
|
||||
match self.queue.write() {
|
||||
Ok(mut queue) => {
|
||||
queue.extend(buf);
|
||||
},
|
||||
Err(_) => {
|
||||
unreachable!("This shouldn't happen (extend)");
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn drain(&self) -> Vec<f32> {
|
||||
match self.queue.write() {
|
||||
Ok(mut queue) => {
|
||||
return queue.drain(..)
|
||||
.collect::<Vec<_>>();
|
||||
},
|
||||
Err(_) => {
|
||||
unreachable!("This shouldn't happen (drain)");
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn drain_size(&self, size: usize) -> Option<Vec<f32>> {
|
||||
match self.queue.write() {
|
||||
Ok(mut queue) => {
|
||||
if queue.len() < size {
|
||||
return None;
|
||||
}
|
||||
let vec = queue.drain(0..size)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
return Some(vec);
|
||||
},
|
||||
Err(_) => {
|
||||
unreachable!("This shouldn't happen (drain_size)");
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
match self.queue.read() {
|
||||
Ok(mut queue) => {
|
||||
return queue.len();
|
||||
},
|
||||
Err(_) => {
|
||||
unreachable!("This shouldn't happen (len)");
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
//!
|
||||
//! This is a process that captures microphone input, ships proto-over-zeromq to a python
|
||||
//! CycleGAN-VC (tensorflow) sidecar server, collects the results, then plays them over
|
||||
//! the speaker.
|
||||
//!
|
||||
|
||||
extern crate bytes;
|
||||
extern crate byteorder;
|
||||
extern crate cpal;
|
||||
extern crate failure;
|
||||
extern crate prost;
|
||||
extern crate prost_types;
|
||||
extern crate wavy;
|
||||
extern crate world_sys;
|
||||
extern crate zmq;
|
||||
|
||||
//include!(concat!(env!("OUT_DIR"), "/voder.audio.rs"));
|
||||
|
||||
pub mod audio_hardware;
|
||||
pub mod audio_queue;
|
||||
//pub mod model;
|
||||
pub mod protos;
|
||||
pub mod sidecar;
|
||||
pub mod synthesis;
|
||||
|
||||
use audio_hardware::AudioHardware;
|
||||
use protos::voder_audio::VocodeAudioRequest;
|
||||
use protos::voder_audio::VocodeAudioResponse;
|
||||
use sidecar::Sidecar;
|
||||
|
||||
use wavy::*;
|
||||
|
||||
use byteorder::{ByteOrder, BigEndian, LittleEndian, ReadBytesExt};
|
||||
use failure::_core::time::Duration;
|
||||
use audio_queue::AudioQueue;
|
||||
use prost::Message;
|
||||
//use model::load_model;
|
||||
//use model::print_version;
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Cursor};
|
||||
use std::path::Path;
|
||||
use std::process::exit;
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
use zmq::{Error, Socket};
|
||||
use protos::voder_audio::vocode_audio_request::VocodeParams;
|
||||
|
||||
fn main() {
|
||||
//print_version();
|
||||
//load_model(); // TODO: This works. Temporarily commented out
|
||||
run_audio().expect("Should work");
|
||||
}
|
||||
|
||||
const LATENCY_MS: f32 = 50.0;
|
||||
|
||||
fn run_audio() -> Result<(), failure::Error> {
|
||||
let mut microphone_queue = Arc::new(AudioQueue::new());
|
||||
let mut microphone_queue_2 = microphone_queue.clone();
|
||||
|
||||
let mut post_process_queue = Arc::new(AudioQueue::new());
|
||||
let mut post_process_queue_2 = post_process_queue.clone();
|
||||
let mut sidecar = Sidecar::new("tcp://127.0.0.1:5555", microphone_queue_2, post_process_queue);
|
||||
|
||||
thread::spawn(move || {
|
||||
sidecar.run();
|
||||
});
|
||||
|
||||
let mut audio_hardware = AudioHardware::new(microphone_queue, post_process_queue_2).expect("");
|
||||
audio_hardware.run();
|
||||
Ok(())
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
pub mod voder_audio;
|
||||
@@ -1,70 +0,0 @@
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct VocodeAudioRequest {
|
||||
#[prost(float, repeated, tag="1")]
|
||||
pub float_audio: ::std::vec::Vec<f32>,
|
||||
/// An autoincrement sent with each request.
|
||||
#[prost(int64, tag="2")]
|
||||
pub request_batch_number: i64,
|
||||
#[prost(message, optional, tag="10")]
|
||||
pub vocode_params: ::std::option::Option<vocode_audio_request::VocodeParams>,
|
||||
// We'll resend these params for every batch, but will only use
|
||||
// the last batch's settings. Kind of a dumb API, but it'll work.
|
||||
//int32 sample_rate = 2;
|
||||
|
||||
///bool skip_resample = 3;
|
||||
#[prost(bool, tag="4")]
|
||||
pub skip_vocode: bool,
|
||||
// Debugging
|
||||
//bool save_files = 5;
|
||||
|
||||
/// How big we let the buffer grow before running 'convert'.
|
||||
#[prost(int32, tag="6")]
|
||||
pub buffer_size_minimum: i32,
|
||||
// Requested output rate
|
||||
//int32 output_rate = 7;
|
||||
|
||||
/// Discard the vocoded audio and return the original
|
||||
#[prost(bool, tag="8")]
|
||||
pub discard_vocoded_audio: bool,
|
||||
}
|
||||
pub mod vocode_audio_request {
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct VocodeParams {
|
||||
/// The initial sample rate coming from the microphone.
|
||||
#[prost(int32, tag="1")]
|
||||
pub initial_sample_rate: i32,
|
||||
/// Original sample
|
||||
#[prost(int32, tag="9")]
|
||||
pub original_source_rate: i32,
|
||||
#[prost(bool, tag="10")]
|
||||
pub original_source_save_file: bool,
|
||||
/// Resample before passing to the algorithm?
|
||||
#[prost(bool, tag="2")]
|
||||
pub pre_convert_resample: bool,
|
||||
#[prost(int32, tag="3")]
|
||||
pub pre_convert_resample_rate: i32,
|
||||
#[prost(bool, tag="4")]
|
||||
pub pre_convert_resample_save_file: bool,
|
||||
/// The hard-coded hyperparameter of the model.
|
||||
/// Probably don't want to change from 16000.
|
||||
#[prost(int32, tag="5")]
|
||||
pub model_hyperparameter_sampling_rate: i32,
|
||||
#[prost(bool, tag="11")]
|
||||
pub model_save_file: bool,
|
||||
/// Resample after passing to the algorithm?
|
||||
#[prost(bool, tag="6")]
|
||||
pub post_convert_resample: bool,
|
||||
#[prost(int32, tag="7")]
|
||||
pub post_convert_resample_rate: i32,
|
||||
#[prost(bool, tag="8")]
|
||||
pub post_convert_resample_save_file: bool,
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct VocodeAudioResponse {
|
||||
#[prost(float, repeated, tag="1")]
|
||||
pub float_audio: ::std::vec::Vec<f32>,
|
||||
/// A sidecar-side autoincrement sent with each request.
|
||||
#[prost(int64, tag="2")]
|
||||
pub response_batch_number: i64,
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
use audio_queue::AudioQueue;
|
||||
use prost::Message;
|
||||
use protos::voder_audio::VocodeAudioRequest;
|
||||
use protos::voder_audio::VocodeAudioResponse;
|
||||
use protos::voder_audio::vocode_audio_request::VocodeParams;
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
use zmq::{Error, Socket};
|
||||
use zmq;
|
||||
|
||||
/// Talks a custom protobuf over ZeroMQ protocol to a Python TensorFlow sidecar process
|
||||
/// that evaluates the model and ships back audio byte results.
|
||||
pub struct Sidecar {
|
||||
socket_endpoint: String,
|
||||
microphone_queue: Arc<AudioQueue>,
|
||||
processed_queue: Arc<AudioQueue>,
|
||||
}
|
||||
|
||||
impl Sidecar {
|
||||
pub fn new(socket_endpoint: &str,
|
||||
microphone_queue: Arc<AudioQueue>,
|
||||
processed_queue: Arc<AudioQueue>) -> Self {
|
||||
Self {
|
||||
socket_endpoint: socket_endpoint.into(),
|
||||
microphone_queue,
|
||||
processed_queue,
|
||||
}
|
||||
}
|
||||
|
||||
/// Heuristic to determine if a speaker is speaking.
|
||||
/// Basically just checks audio loudness.
|
||||
fn is_speaking(audio: &Vec<f32>) -> bool {
|
||||
let mut avg_pos = 0.0f32;
|
||||
let mut num_pos = 0;
|
||||
|
||||
for x in audio {
|
||||
if x > &0.0f32 {
|
||||
avg_pos += x;
|
||||
num_pos += 1;
|
||||
}
|
||||
}
|
||||
|
||||
avg_pos = avg_pos / num_pos as f32;
|
||||
avg_pos > 0.006f32
|
||||
}
|
||||
|
||||
pub fn run(&mut self) {
|
||||
let mut context = zmq::Context::new();
|
||||
let mut socket = context.socket(zmq::REQ).unwrap();
|
||||
|
||||
socket.connect(&self.socket_endpoint).unwrap();
|
||||
|
||||
let mut reconnect = false;
|
||||
let mut fail_count = 0;
|
||||
|
||||
const SEND_SIZE : usize = 5000;
|
||||
|
||||
let mut request_batch_number = 0i64;
|
||||
|
||||
let mut activated = false;
|
||||
let mut frames_activated = 0;
|
||||
let mut frames_deactivated = 0;
|
||||
|
||||
let mut ring_buffer = VecDeque::with_capacity(5);
|
||||
|
||||
loop {
|
||||
let mut drained = match self.microphone_queue.drain_size(SEND_SIZE) {
|
||||
None => { continue; },
|
||||
Some(d) => d,
|
||||
};
|
||||
|
||||
/*
|
||||
Heuristic for audio. Don't record when silent (using threshold).
|
||||
|
||||
mic: act act act dec act act act act dec dec dec dec dec act dec
|
||||
sent: | ok ok ok ok ok ok ok ok ok ok ok|
|
||||
*/
|
||||
|
||||
|
||||
// NB: We don't want to lose audio right at the activating edge.
|
||||
// Save some bounded history to replay once activated.
|
||||
ring_buffer.push_back(drained.clone());
|
||||
if ring_buffer.len() > 5 {
|
||||
ring_buffer.pop_front();
|
||||
}
|
||||
|
||||
if activated {
|
||||
if !Self::is_speaking(&drained) {
|
||||
frames_deactivated += 1;
|
||||
} else {
|
||||
frames_deactivated = 0;
|
||||
}
|
||||
|
||||
if frames_deactivated > 20 {
|
||||
println!("Deactivating edge");
|
||||
activated = false;
|
||||
frames_activated = 0;
|
||||
frames_deactivated = 0;
|
||||
}
|
||||
|
||||
} else {
|
||||
if Self::is_speaking(&drained) {
|
||||
frames_activated += 1;
|
||||
} else {
|
||||
frames_activated = 0;
|
||||
}
|
||||
|
||||
if frames_activated > 3 {
|
||||
println!("Activating edge");
|
||||
activated = true;
|
||||
frames_activated = 0;
|
||||
frames_deactivated = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if !activated {
|
||||
continue;
|
||||
}
|
||||
|
||||
let drained = ring_buffer.drain(..)
|
||||
.flat_map(|sample| sample)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
request_batch_number += 1;
|
||||
|
||||
let mut vocode_request = VocodeAudioRequest::default();
|
||||
//vocode_request.sample_rate = 16000;
|
||||
//vocode_request.output_rate = 16000;
|
||||
//vocode_request.model_sampling_rate = 88000; // This is close!
|
||||
//vocode_request.model_sampling_rate = 88000;
|
||||
//vocode_request.skip_resample = false;
|
||||
vocode_request.skip_vocode = false;
|
||||
vocode_request.discard_vocoded_audio = false;
|
||||
//vocode_request.buffer_size_minimum = 5000; // AWFUL. SO CHOPPY.
|
||||
//vocode_request.buffer_size_minimum = 50000; // Practically real time, but lots more phase distortion.
|
||||
//vocode_request.buffer_size_minimum = 30000; // Hm, hmm... not bad
|
||||
vocode_request.buffer_size_minimum = 70000; // Sounds pretty good.
|
||||
//vocode_request.buffer_size_minimum = 100000; // This sounds good! A bit slow ~3seconds.
|
||||
//vocode_request.buffer_size_minimum = 200000;
|
||||
vocode_request.float_audio = drained.clone();
|
||||
vocode_request.request_batch_number = request_batch_number;
|
||||
|
||||
let mut vocode_params = VocodeParams::default();
|
||||
vocode_params.original_source_rate = 88000; // This is correct for saving
|
||||
|
||||
vocode_params.pre_convert_resample = true;
|
||||
vocode_params.pre_convert_resample_rate = 16000;
|
||||
|
||||
vocode_params.model_hyperparameter_sampling_rate = 16000;
|
||||
|
||||
vocode_params.post_convert_resample = true;
|
||||
vocode_params.post_convert_resample_rate = 88000;
|
||||
|
||||
vocode_params.original_source_save_file = false;
|
||||
vocode_params.pre_convert_resample_save_file = false;
|
||||
vocode_params.model_save_file = false;
|
||||
vocode_params.post_convert_resample_save_file = false;
|
||||
|
||||
vocode_request.vocode_params = Some(vocode_params);
|
||||
|
||||
let mut encoded_bytes = Vec::with_capacity(vocode_request.encoded_len());
|
||||
vocode_request.encode(&mut encoded_bytes).unwrap();
|
||||
|
||||
if reconnect {
|
||||
reconnect = false;
|
||||
println!("RECONNECT");
|
||||
|
||||
//thread::sleep(Duration::from_millis(200));
|
||||
|
||||
socket = match context.socket(zmq::REQ) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
println!("Error creating socket: {:?}", e);
|
||||
continue
|
||||
},
|
||||
};
|
||||
|
||||
match socket.connect("tcp://127.0.0.1:5555") {
|
||||
Ok(_) => {},
|
||||
Err(err) => {
|
||||
println!("Err B: {:?}", err);
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
match socket.send(&encoded_bytes, 0) {
|
||||
Ok(_) => {
|
||||
//println!("Sent len: {}", bytes.len());
|
||||
},
|
||||
Err(e) => {
|
||||
println!("send err {}: {:?}", e.to_raw(), e);
|
||||
fail_count += 1;
|
||||
},
|
||||
}
|
||||
|
||||
match socket.recv_bytes(0) {
|
||||
Ok(buf) => {
|
||||
//println!("---> Buf len: {}", buf.len());
|
||||
if buf.len() > 2 {
|
||||
// Receive data condition.
|
||||
let vocode_response = VocodeAudioResponse::decode(buf).unwrap();
|
||||
self.processed_queue.extend(vocode_response.float_audio);
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
fail_count += 1;
|
||||
println!("recv err: {:?}", e);
|
||||
},
|
||||
}
|
||||
|
||||
if fail_count > 5 {
|
||||
fail_count = 0;
|
||||
reconnect = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
use world_sys::core::synthesize::SynthesizeError;
|
||||
use world_sys::core::synthesize::SynthesizeResult;
|
||||
use world_sys::core::synthesize::synthesize;
|
||||
|
||||
use world_sys::core::spectral::code_spectral_envelope;
|
||||
|
||||
/*
|
||||
TODO: ALGORITHM TO GET VOICE TRANSFORMATION:
|
||||
|
||||
- [mono]
|
||||
- [librosa.resample (downsample)]
|
||||
- core convert()
|
||||
- wav_padding()
|
||||
- [maths]
|
||||
- [np.pad]
|
||||
- world_decompose()
|
||||
- pyworld.harvest
|
||||
- pyworld.cheaptrick
|
||||
- pyworld.d4c
|
||||
- world_encode_spectral_envelope()
|
||||
- pyworld.code_spectral_envelope
|
||||
- [matrix transpose]
|
||||
- pitch_conversion()
|
||||
- [maths]
|
||||
- [maths : coded_sp_norm]
|
||||
- << evaluate ML model >>
|
||||
- [maths : coded_sp_converted]
|
||||
- [matrix transpose]
|
||||
- [np.ascontiguousarray]
|
||||
- world_decode_spectral_envelop
|
||||
- pyworld.get_cheaptrick_fft_size
|
||||
- pyworld.decode_spectral_envelope
|
||||
- world_speech_synthesis
|
||||
- pyworld.synthesize
|
||||
- [librosa.resample (upsample)]
|
||||
- [wav things]
|
||||
*/
|
||||
|
||||
Reference in new issue
Block a user