From fb877bc290e4e3b97b45b6bb853619c35f478c7c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 28 Aug 2021 23:15:29 -0400 Subject: [PATCH] ControllableTalkNet from https://github.com/SortAnon/ControllableTalkNet --- .../tacotron2/ControllableTalkNet/Dockerfile | 29 + models/tacotron2/ControllableTalkNet/LICENSE | 121 ++ .../tacotron2/ControllableTalkNet/README.md | 26 + .../TalkNet_Training_Offline.ipynb | 925 ++++++++++++++ .../ControllableTalkNet/assets/Equestria.woff | Bin 0 -> 22008 bytes .../assets/dash_customfont.css | 420 +++++++ .../controllable_talknet.py | 1097 +++++++++++++++++ .../ControllableTalkNet/diffsvc_gui.py | 368 ++++++ .../model_lists/sortanon_models.json | 165 +++ .../ControllableTalkNet/mycroft_talknet.py | 114 ++ .../ControllableTalkNet/requirements.txt | 28 + .../ControllableTalkNet/talknet_offline.py | 17 + .../ControllableTalkNet/talknet_singer.py | 225 ++++ 13 files changed, 3535 insertions(+) create mode 100644 models/tacotron2/ControllableTalkNet/Dockerfile create mode 100644 models/tacotron2/ControllableTalkNet/LICENSE create mode 100644 models/tacotron2/ControllableTalkNet/README.md create mode 100644 models/tacotron2/ControllableTalkNet/TalkNet_Training_Offline.ipynb create mode 100644 models/tacotron2/ControllableTalkNet/assets/Equestria.woff create mode 100644 models/tacotron2/ControllableTalkNet/assets/dash_customfont.css create mode 100644 models/tacotron2/ControllableTalkNet/controllable_talknet.py create mode 100644 models/tacotron2/ControllableTalkNet/diffsvc_gui.py create mode 100644 models/tacotron2/ControllableTalkNet/model_lists/sortanon_models.json create mode 100644 models/tacotron2/ControllableTalkNet/mycroft_talknet.py create mode 100644 models/tacotron2/ControllableTalkNet/requirements.txt create mode 100644 models/tacotron2/ControllableTalkNet/talknet_offline.py create mode 100644 models/tacotron2/ControllableTalkNet/talknet_singer.py diff --git a/models/tacotron2/ControllableTalkNet/Dockerfile b/models/tacotron2/ControllableTalkNet/Dockerfile new file mode 100644 index 0000000000..be6e6b1481 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/Dockerfile @@ -0,0 +1,29 @@ +FROM nvidia/cuda:11.0.3-base-ubuntu20.04 +ENV TZ=Etc/GMT +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN apt-get update && apt-get install -y git ffmpeg python3.8 python3-pip curl +RUN git clone -q https://github.com/SortAnon/ControllableTalkNet /talknet +RUN git clone -q https://github.com/SortAnon/hifi-gan /talknet/hifi-gan +RUN python3.8 -m pip --no-cache-dir install -r "/talknet/requirements.txt" -f https://download.pytorch.org/whl/torch_stable.html +RUN python3.8 -m pip --no-cache-dir install git+https://github.com/SortAnon/NeMo.git +RUN python3.8 -m pip uninstall -y pesq +RUN python3.8 -m pip install pesq==0.0.2 +RUN touch /talknet/is_docker +WORKDIR /talknet +EXPOSE 8050 + +RUN printf "#!/bin/bash \ + \necho Updating TalkNet... \ + \ngit -C /talknet reset --hard origin/main -q \ + \ngit -C /talknet pull origin main -q \ + \necho Updating HiFi-GAN... \ + \ngit -C /talknet reset --hard origin/main -q \ + \ngit -C /talknet pull origin main -q \ + \necho Updating Python dependencies... \ + \npython3.8 -m pip --quiet --no-cache-dir install -r /talknet/requirements.txt -f https://download.pytorch.org/whl/torch_stable.html \ + \npython3.8 -m pip --quiet --no-cache-dir install git+https://github.com/SortAnon/NeMo.git \ + \necho Launching TalkNet... \ + \npython3.8 talknet_offline.py\n" > /talknet/docker_launch.sh + +RUN chmod +x /talknet/docker_launch.sh +CMD ["./docker_launch.sh"] diff --git a/models/tacotron2/ControllableTalkNet/LICENSE b/models/tacotron2/ControllableTalkNet/LICENSE new file mode 100644 index 0000000000..0e259d42c9 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/LICENSE @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/models/tacotron2/ControllableTalkNet/README.md b/models/tacotron2/ControllableTalkNet/README.md new file mode 100644 index 0000000000..20f0d0429a --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/README.md @@ -0,0 +1,26 @@ +# Controllable TalkNet +Controllable TalkNet is a web application that lets you synthesize speech, +which mimics the pitch and pacing of an existing audio clip. It's based on [NVIDIA's implementation](https://github.com/NVIDIA/NeMo) +of TalkNet 2, with some changes to support singing synthesis and higher audio quality. + +## Requirements +* A Google account to run Colab, or... +* An NVIDIA GPU with 4+ GB of VRAM +* 10 GB of free space + +## How to run +### Google Colab +* [Go to the Colab notebook](https://colab.research.google.com/drive/1aj6Jk8cpRw7SsN3JSYCv57CrR6s0gYPB) and follow the instructions. + +### TalkNet Offline (Windows) +* [Download the setup script](https://github.com/SortAnon/ControllableTalkNet/releases/latest/download/TalkNetOffline.zip) +and extract it to a folder. +* Run setup.bat. The initial setup will take about 20 minutes. +* When it's done, run talknet.bat to start TalkNet on http://127.0.0.1:8050/. To download updates, run update.bat. + +### Docker (Linux) +* Install Docker and NVIDIA Container Toolkit. +* [Download the Dockerfile.](https://raw.githubusercontent.com/SortAnon/ControllableTalkNet/main/Dockerfile) +Open a terminal, and navigate to the directory where you saved it. +* Run ```docker build -t talknet-offline .``` to build the image. Add ```sudo``` if you're not using rootless Docker. +* Run ```docker run -it --gpus all -p 8050:8050 talknet-offline``` to start TalkNet on http://127.0.0.1:8050/. diff --git a/models/tacotron2/ControllableTalkNet/TalkNet_Training_Offline.ipynb b/models/tacotron2/ControllableTalkNet/TalkNet_Training_Offline.ipynb new file mode 100644 index 0000000000..44b8cda03f --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/TalkNet_Training_Offline.ipynb @@ -0,0 +1,925 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# TalkNet Training (offline)\n", + "\n", + "To train a 22KHz TalkNet, run the cells below and follow the instructions.\n", + "\n", + "The notebook will automatically resume training any models from the last saved checkpoint. If you're resuming from a new session, always re-run step 1 first." + ], + "metadata": { + "id": "Gss5Ox_RNiba" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 1:** Configure training data paths. Create the following and change the paths below:\n", + "* A dataset of .wav files, packaged as a .zip or .tar file\n", + "* Training and validation filelists, in LJSpeech format with relative paths (note: ARPABET transcripts are not supported)\n", + "* An output path for checkpoints" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "### CHANGE THESE ###\n", + "\n", + "dataset = \"/absolute/path_to_dataset.zip\"\n", + "train_filelist = \"/absolute/train_filelist.txt\"\n", + "val_filelist = \"/absolute/val_filelist.txt\"\n", + "output_dir = \"/absolute/name_of_character\"\n", + "\n", + "### ------------ ###\n", + "\n", + "import os\n", + "import gdown\n", + "import platform\n", + "\n", + "assert os.path.exists(dataset), \"Cannot find dataset\"\n", + "assert os.path.exists(train_filelist), \"Cannot find training filelist\"\n", + "assert os.path.exists(val_filelist), \"Cannot find validation filelist\"\n", + "if not os.path.exists(output_dir):\n", + " os.makedirs(output_dir)\n", + "\n", + "if os.getcwd() != output_dir:\n", + " cwd = os.getcwd()\n", + "else:\n", + " os.chdir(cwd)\n", + " \n", + "# Download pre-trained models\n", + "if not os.path.exists(\"talknet_spect.nemo\"):\n", + " print(\"Downloading pre-trained models...\")\n", + " zip_path = \"TalkNet_Pretrained.zip\"\n", + " if not os.path.exists(zip_path) or os.stat(zip_path).st_size < 100:\n", + " d = 'https://drive.google.com/uc?id='\n", + " gdown.download(d+\"19wSym9mNEnmzLS9XdPlfNAW9_u-mP1hR\", zip_path, quiet=False)\n", + " print(platform.system())\n", + " if \"Windows\" in platform.system():\n", + " !tar -xf {zip_path}\n", + " else:\n", + " !unzip -qo {zip_path}\n", + " os.remove(zip_path)\n", + "\n", + "print(\"OK\")" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "nfSawDUD5tqv" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 2:** Dataset processing, part 1.\n", + "\n", + "If this step fails, try the following:\n", + "* Make sure your filelists are correct. They should have relative \n", + "paths that match the contents of the archive." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "import os\n", + "import shutil\n", + "import sys\n", + "import json\n", + "import nemo\n", + "import torch\n", + "import torchaudio\n", + "import numpy as np\n", + "#from pysptk import sptk\n", + "from pathlib import Path\n", + "from tqdm.notebook import tqdm\n", + "import ffmpeg\n", + "\n", + "def fix_transcripts(inpath):\n", + " found_arpabet = False\n", + " found_grapheme = False\n", + " with open(inpath, \"r\", encoding=\"utf8\") as f:\n", + " lines = f.readlines()\n", + " with open(inpath, \"w\", encoding=\"utf8\") as f:\n", + " for l in lines:\n", + " if l.strip() == \"\":\n", + " continue\n", + " if \"{\" in l:\n", + " if not found_arpabet:\n", + " print(\"Warning: Skipping ARPABET lines (not supported).\")\n", + " found_arpabet = True\n", + " else:\n", + " f.write(l)\n", + " found_grapheme = True\n", + " assert found_grapheme, \"No non-ARPABET lines found in \" + inpath\n", + "\n", + "def generate_json(inpath, outpath):\n", + " output = \"\"\n", + " sample_rate = 22050\n", + " with open(inpath, \"r\", encoding=\"utf8\") as f:\n", + " for l in f.readlines():\n", + " lpath = l.split(\"|\")[0].strip()\n", + " if lpath[:5] != \"wavs/\":\n", + " lpath = \"wavs/\" + lpath\n", + " lpath = os.path.join(output_dir, \"_training_data\", lpath)\n", + " size = os.stat(lpath).st_size\n", + " x = {\n", + " \"audio_filepath\": lpath,\n", + " \"duration\": size / (sample_rate * 2),\n", + " \"text\": l.split(\"|\")[1].strip(),\n", + " }\n", + " output += json.dumps(x) + \"\\n\"\n", + " with open(outpath, \"w\", encoding=\"utf8\") as w:\n", + " w.write(output)\n", + "\n", + "def convert_to_22k(inpath):\n", + " if inpath.strip()[-4:].lower() != \".wav\":\n", + " print(\"Warning: \" + inpath.strip() + \" is not a .wav file!\")\n", + " return\n", + " ffmpeg.input(inpath).output(\n", + " inpath + \"_22k.wav\",\n", + " ar=\"22050\",\n", + " ac=\"1\",\n", + " acodec=\"pcm_s16le\",\n", + " map_metadata=\"-1\",\n", + " fflags=\"+bitexact\",\n", + " ).overwrite_output().run(quiet=True)\n", + " os.remove(inpath)\n", + " os.rename(inpath + \"_22k.wav\", inpath)\n", + "\n", + "# Extract dataset\n", + "os.chdir(output_dir)\n", + "\n", + "if os.path.exists(\"_training_data\"):\n", + " shutil.rmtree(\"_training_data\")\n", + "os.mkdir(\"_training_data\")\n", + "os.chdir(\"_training_data\")\n", + "os.mkdir(\"wavs\")\n", + "os.chdir(\"wavs\")\n", + "if dataset[-4:] == \".zip\":\n", + " if \"Windows\" in platform.system():\n", + " !tar -xf \"{dataset}\"\n", + " else:\n", + " !unzip -qo \"{dataset}\"\n", + "elif dataset[-4:] == \".tar\":\n", + " !tar -xf \"{dataset}\"\n", + "else:\n", + " raise Exception(\"Unknown extension for dataset\")\n", + "if os.path.exists(os.path.join(output_dir, \"_training_data\", \"wavs\", \"wavs\")):\n", + " shutil.move(\n", + " os.path.join(output_dir, \"_training_data\", \"wavs\", \"wavs\"), \n", + " os.path.join(output_dir, \"_training_data\", \"tempwavs\")\n", + " )\n", + " shutil.rmtree(os.path.join(output_dir, \"_training_data\", \"wavs\"))\n", + " shutil.move(\n", + " os.path.join(output_dir, \"_training_data\", \"tempwavs\"), \n", + " os.path.join(output_dir, \"_training_data\", \"wavs\")\n", + " )\n", + "# Filelist for preprocessing\n", + "os.chdir(output_dir)\n", + "shutil.copy(train_filelist, \"trainfiles.txt\")\n", + "shutil.copy(val_filelist, \"valfiles.txt\")\n", + "fix_transcripts(\"trainfiles.txt\")\n", + "fix_transcripts(\"valfiles.txt\")\n", + "seen_files = []\n", + "with open(\"trainfiles.txt\") as f:\n", + " t = f.read().split(\"\\n\")\n", + "with open(\"valfiles.txt\") as f:\n", + " v = f.read().split(\"\\n\")\n", + " all_filelist = t[:] + v[:]\n", + "with open(\"allfiles.txt\", \"w\") as f:\n", + " for x in all_filelist:\n", + " if x.strip() == \"\":\n", + " continue\n", + " if x.split(\"|\")[0] not in seen_files:\n", + " seen_files.append(x.split(\"|\")[0])\n", + " f.write(x.strip() + \"\\n\")\n", + "\n", + "# Ensure audio is 22k\n", + "print(\"Converting audio...\")\n", + "for r, _, f in os.walk(os.path.join(\"_training_data\", \"wavs\")):\n", + " for name in tqdm(f):\n", + " convert_to_22k(os.path.join(r, name))\n", + "\n", + "# Convert to JSON\n", + "generate_json(\"trainfiles.txt\", \"trainfiles.json\")\n", + "generate_json(\"valfiles.txt\", \"valfiles.json\")\n", + "generate_json(\"allfiles.txt\", \"allfiles.json\")\n", + "\n", + "print(\"OK\")" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "bxFr3Fdi_kOC" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 3:** Dataset processing, part 2. This takes a while, but\n", + "you only have to run this once per dataset (results are saved to Drive).\n", + "\n", + "If this step fails, try the following:\n", + "* Make sure your dataset only contains WAV files." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "# Extract phoneme duration\n", + "\n", + "import json\n", + "from nemo.collections.asr.models import EncDecCTCModel\n", + "asr_model = EncDecCTCModel.from_pretrained(model_name=\"asr_talknet_aligner\").cpu().eval()\n", + "\n", + "def forward_extractor(tokens, log_probs, blank):\n", + " \"\"\"Computes states f and p.\"\"\"\n", + " n, m = len(tokens), log_probs.shape[0]\n", + " # `f[s, t]` -- max sum of log probs for `s` first codes\n", + " # with `t` first timesteps with ending in `tokens[s]`.\n", + " f = np.empty((n + 1, m + 1), dtype=float)\n", + " f.fill(-(10 ** 9))\n", + " p = np.empty((n + 1, m + 1), dtype=int)\n", + " f[0, 0] = 0.0 # Start\n", + " for s in range(1, n + 1):\n", + " c = tokens[s - 1]\n", + " for t in range((s + 1) // 2, m + 1):\n", + " f[s, t] = log_probs[t - 1, c]\n", + " # Option #1: prev char is equal to current one.\n", + " if s == 1 or c == blank or c == tokens[s - 3]:\n", + " options = f[s : (s - 2 if s > 1 else None) : -1, t - 1]\n", + " else: # Is not equal to current one.\n", + " options = f[s : (s - 3 if s > 2 else None) : -1, t - 1]\n", + " f[s, t] += np.max(options)\n", + " p[s, t] = np.argmax(options)\n", + " return f, p\n", + "\n", + "\n", + "def backward_extractor(f, p):\n", + " \"\"\"Computes durs from f and p.\"\"\"\n", + " n, m = f.shape\n", + " n -= 1\n", + " m -= 1\n", + " durs = np.zeros(n, dtype=int)\n", + " if f[-1, -1] >= f[-2, -1]:\n", + " s, t = n, m\n", + " else:\n", + " s, t = n - 1, m\n", + " while s > 0:\n", + " durs[s - 1] += 1\n", + " s -= p[s, t]\n", + " t -= 1\n", + " assert durs.shape[0] == n\n", + " assert np.sum(durs) == m\n", + " assert np.all(durs[1::2] > 0)\n", + " return durs\n", + "\n", + "def preprocess_tokens(tokens, blank):\n", + " new_tokens = [blank]\n", + " for c in tokens:\n", + " new_tokens.extend([c, blank])\n", + " tokens = new_tokens\n", + " return tokens\n", + "\n", + "data_config = {\n", + " 'manifest_filepath': \"allfiles.json\",\n", + " 'sample_rate': 22050,\n", + " 'labels': asr_model.decoder.vocabulary,\n", + " 'batch_size': 1,\n", + "}\n", + "\n", + "parser = nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset.make_vocab(\n", + " notation='phonemes', punct=True, spaces=True, stresses=False, add_blank_at=\"last\"\n", + ")\n", + "\n", + "dataset = nemo.collections.asr.data.audio_to_text._AudioTextDataset(\n", + " manifest_filepath=data_config['manifest_filepath'], sample_rate=data_config['sample_rate'], parser=parser,\n", + ")\n", + "\n", + "dl = torch.utils.data.DataLoader(\n", + " dataset=dataset, batch_size=data_config['batch_size'], collate_fn=dataset.collate_fn, shuffle=False,\n", + ")\n", + "\n", + "blank_id = asr_model.decoder.num_classes_with_blank - 1\n", + "\n", + "if os.path.exists(os.path.join(output_dir, \"durations.pt\")):\n", + " print(\"durations.pt already exists; skipping\")\n", + "else:\n", + " dur_data = {}\n", + " for sample_idx, test_sample in tqdm(enumerate(dl), total=len(dl)):\n", + " log_probs, _, greedy_predictions = asr_model(\n", + " input_signal=test_sample[0], input_signal_length=test_sample[1]\n", + " )\n", + "\n", + " log_probs = log_probs[0].cpu().detach().numpy()\n", + " seq_ids = test_sample[2][0].cpu().detach().numpy()\n", + "\n", + " target_tokens = preprocess_tokens(seq_ids, blank_id)\n", + "\n", + " f, p = forward_extractor(target_tokens, log_probs, blank_id)\n", + " durs = backward_extractor(f, p)\n", + "\n", + " dur_key = Path(dl.dataset.collection[sample_idx].audio_file).stem\n", + " dur_data[dur_key] = {\n", + " 'blanks': torch.tensor(durs[::2], dtype=torch.long).cpu().detach(), \n", + " 'tokens': torch.tensor(durs[1::2], dtype=torch.long).cpu().detach()\n", + " }\n", + "\n", + " del test_sample\n", + "\n", + " torch.save(dur_data, os.path.join(output_dir, \"durations.pt\"))\n", + "\n", + "#Extract F0 (pitch)\n", + "import crepe\n", + "from scipy.io import wavfile\n", + "\n", + "def crepe_f0(audio_file, hop_length=256):\n", + " sr, audio = wavfile.read(audio_file)\n", + " audio_x = np.arange(0, len(audio)) / 22050.0\n", + " time, frequency, confidence, activation = crepe.predict(audio, sr, viterbi=True)\n", + "\n", + " x = np.arange(0, len(audio), hop_length) / 22050.0\n", + " freq_interp = np.interp(x, time, frequency)\n", + " conf_interp = np.interp(x, time, confidence)\n", + " audio_interp = np.interp(x, audio_x, np.absolute(audio)) / 32768.0\n", + " weights = [0.5, 0.25, 0.25]\n", + " audio_smooth = np.convolve(audio_interp, np.array(weights)[::-1], \"same\")\n", + "\n", + " conf_threshold = 0.25\n", + " audio_threshold = 0.0005\n", + " for i in range(len(freq_interp)):\n", + " if conf_interp[i] < conf_threshold:\n", + " freq_interp[i] = 0.0\n", + " if audio_smooth[i] < audio_threshold:\n", + " freq_interp[i] = 0.0\n", + "\n", + " # Hack to make f0 and mel lengths equal\n", + " if len(audio) % hop_length == 0:\n", + " freq_interp = np.pad(freq_interp, pad_width=[0, 1])\n", + " return torch.from_numpy(freq_interp.astype(np.float32))\n", + "\n", + "if os.path.exists(os.path.join(output_dir, \"f0s.pt\")):\n", + " print(\"f0s.pt already exists; skipping\")\n", + "else:\n", + " f0_data = {}\n", + " with open(\"allfiles.json\") as f:\n", + " for i, l in enumerate(f.readlines()):\n", + " print(str(i))\n", + " audio_path = json.loads(l)[\"audio_filepath\"]\n", + " f0_data[Path(audio_path).stem] = crepe_f0(audio_path)\n", + "\n", + " # calculate f0 stats (mean & std) only for train set\n", + " with open(\"trainfiles.json\") as f:\n", + " train_ids = {Path(json.loads(l)[\"audio_filepath\"]).stem for l in f}\n", + " all_f0 = torch.cat([f0[f0 >= 1e-5] for f0_id, f0 in f0_data.items() if f0_id in train_ids])\n", + "\n", + " F0_MEAN, F0_STD = all_f0.mean().item(), all_f0.std().item() \n", + " print(\"F0_MEAN: \" + str(F0_MEAN) + \", F0_STD: \" + str(F0_STD))\n", + " torch.save(f0_data, os.path.join(output_dir, \"f0s.pt\"))\n", + " with open(os.path.join(output_dir, \"f0_info.json\"), \"w\") as f:\n", + " f.write(json.dumps({\"FO_MEAN\": F0_MEAN, \"F0_STD\": F0_STD}))\n", + "\n", + "print(\"OK\")" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "sos9vsxPkIN7" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 4:** Train duration predictor.\n", + "\n", + "If CUDA runs out of memory, try the following:\n", + "* Click on Kernel -> Restart, re-run step 1, and try again.\n", + "* If that doesn't help, reduce the batch size (default 64)." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "batch_size = 64\n", + "\n", + "import os\n", + "from hydra.experimental import compose, initialize\n", + "from hydra.core.global_hydra import GlobalHydra\n", + "from omegaconf import OmegaConf\n", + "import pytorch_lightning as pl\n", + "from nemo.collections.common.callbacks import LogEpochTimeCallback\n", + "from nemo.collections.tts.models import TalkNetDursModel\n", + "from nemo.core.config import hydra_runner\n", + "from nemo.utils.exp_manager import exp_manager\n", + "\n", + "epochs = 20\n", + "learning_rate = 1e-3\n", + "min_learning_rate = 3e-6\n", + "load_checkpoints = True\n", + "\n", + "def train(cfg):\n", + " cfg.sample_rate = 22050\n", + " cfg.train_dataset = os.path.join(output_dir, \"trainfiles.json\")\n", + " cfg.validation_datasets = os.path.join(output_dir, \"valfiles.json\")\n", + " cfg.durs_file = os.path.join(output_dir, \"durations.pt\")\n", + " cfg.f0_file = os.path.join(output_dir, \"f0s.pt\")\n", + " cfg.trainer.accelerator = \"dp\"\n", + " cfg.trainer.max_epochs = epochs\n", + " cfg.trainer.check_val_every_n_epoch = 5\n", + " cfg.model.train_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.validation_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.optim.lr = learning_rate\n", + " cfg.model.optim.sched.min_lr = min_learning_rate\n", + " cfg.exp_manager.exp_dir = output_dir\n", + "\n", + " # Find checkpoints\n", + " ckpt_path = \"\"\n", + " if load_checkpoints:\n", + " path0 = os.path.join(output_dir, \"TalkNetDurs\")\n", + " if os.path.exists(path0):\n", + " path1 = sorted(os.listdir(path0))\n", + " for i in range(len(path1)):\n", + " path2 = os.path.join(path0, path1[-(1+i)], \"checkpoints\")\n", + " if os.path.exists(path2):\n", + " match = [x for x in os.listdir(path2) if \"last.ckpt\" in x]\n", + " if len(match) > 0:\n", + " ckpt_path = os.path.join(path2, match[0])\n", + " print(\"Resuming training from \" + match[0])\n", + " break\n", + " \n", + " if ckpt_path != \"\":\n", + " trainer = pl.Trainer(**cfg.trainer, resume_from_checkpoint = ckpt_path)\n", + " model = TalkNetDursModel(cfg=cfg.model, trainer=trainer)\n", + " else:\n", + " warmstart_path = os.path.join(cwd, \"talknet_durs.nemo\")\n", + " trainer = pl.Trainer(**cfg.trainer)\n", + " model = TalkNetDursModel.restore_from(warmstart_path, override_config_path=cfg)\n", + " model.set_trainer(trainer)\n", + " model.setup_training_data(cfg.model.train_ds)\n", + " model.setup_validation_data(cfg.model.validation_ds)\n", + " model.setup_optimization(cfg.model.optim)\n", + " print(\"Warm-starting from \" + warmstart_path)\n", + " exp_manager(trainer, cfg.get('exp_manager', None))\n", + " trainer.callbacks.extend([pl.callbacks.LearningRateMonitor(), LogEpochTimeCallback()]) # noqa\n", + " trainer.fit(model)\n", + "\n", + "os.chdir(cwd)\n", + "GlobalHydra().clear()\n", + "initialize(config_path=\"conf\")\n", + "cfg = compose(config_name=\"talknet-durs\")\n", + "train(cfg)\n" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "nM7-bMpKO7U2" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 5:** Train pitch predictor.\n", + "\n", + "If CUDA runs out of memory, try the following:\n", + "* Click on Kernel -> Restart, re-run step 1, and try again.\n", + "* If that doesn't help, reduce the batch size (default 64)." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "batch_size = 64\n", + "epochs = 50\n", + "\n", + "import json\n", + "\n", + "with open(os.path.join(output_dir, \"f0_info.json\"), \"r\") as f:\n", + " f0_info = json.load(f)\n", + " f0_mean = f0_info[\"FO_MEAN\"]\n", + " f0_std = f0_info[\"F0_STD\"]\n", + "\n", + "learning_rate = 1e-3\n", + "min_learning_rate = 3e-6\n", + "load_checkpoints = True\n", + "\n", + "import os\n", + "from hydra.experimental import compose, initialize\n", + "from hydra.core.global_hydra import GlobalHydra\n", + "from omegaconf import OmegaConf\n", + "import pytorch_lightning as pl\n", + "from nemo.collections.common.callbacks import LogEpochTimeCallback\n", + "from nemo.collections.tts.models import TalkNetPitchModel\n", + "from nemo.core.config import hydra_runner\n", + "from nemo.utils.exp_manager import exp_manager\n", + "\n", + "def train(cfg):\n", + " cfg.sample_rate = 22050\n", + " cfg.train_dataset = os.path.join(output_dir, \"trainfiles.json\")\n", + " cfg.validation_datasets = os.path.join(output_dir, \"valfiles.json\")\n", + " cfg.durs_file = os.path.join(output_dir, \"durations.pt\")\n", + " cfg.f0_file = os.path.join(output_dir, \"f0s.pt\")\n", + " cfg.trainer.accelerator = \"dp\"\n", + " cfg.trainer.max_epochs = epochs\n", + " cfg.trainer.check_val_every_n_epoch = 5\n", + " cfg.model.f0_mean=f0_mean\n", + " cfg.model.f0_std=f0_std\n", + " cfg.model.train_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.validation_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.optim.lr = learning_rate\n", + " cfg.model.optim.sched.min_lr = min_learning_rate\n", + " cfg.exp_manager.exp_dir = output_dir\n", + "\n", + " # Find checkpoints\n", + " ckpt_path = \"\"\n", + " if load_checkpoints:\n", + " path0 = os.path.join(output_dir, \"TalkNetPitch\")\n", + " if os.path.exists(path0):\n", + " path1 = sorted(os.listdir(path0))\n", + " for i in range(len(path1)):\n", + " path2 = os.path.join(path0, path1[-(1+i)], \"checkpoints\")\n", + " if os.path.exists(path2):\n", + " match = [x for x in os.listdir(path2) if \"last.ckpt\" in x]\n", + " if len(match) > 0:\n", + " ckpt_path = os.path.join(path2, match[0])\n", + " print(\"Resuming training from \" + match[0])\n", + " break\n", + " \n", + " if ckpt_path != \"\":\n", + " trainer = pl.Trainer(**cfg.trainer, resume_from_checkpoint = ckpt_path)\n", + " model = TalkNetPitchModel(cfg=cfg.model, trainer=trainer)\n", + " else:\n", + " warmstart_path = os.path.join(cwd, \"talknet_pitch.nemo\")\n", + " trainer = pl.Trainer(**cfg.trainer)\n", + " model = TalkNetPitchModel.restore_from(warmstart_path, override_config_path=cfg)\n", + " model.set_trainer(trainer)\n", + " model.setup_training_data(cfg.model.train_ds)\n", + " model.setup_validation_data(cfg.model.validation_ds)\n", + " model.setup_optimization(cfg.model.optim)\n", + " print(\"Warm-starting from \" + warmstart_path)\n", + " exp_manager(trainer, cfg.get('exp_manager', None))\n", + " trainer.callbacks.extend([pl.callbacks.LearningRateMonitor(), LogEpochTimeCallback()]) # noqa\n", + " trainer.fit(model)\n", + "\n", + "os.chdir(cwd)\n", + "GlobalHydra().clear()\n", + "initialize(config_path=\"conf\")\n", + "cfg = compose(config_name=\"talknet-pitch\")\n", + "train(cfg)" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "JLfm00NuJfon" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 6:** Train spectrogram generator. 200+ epochs are recommended. \n", + "\n", + "This is the slowest of the three models to train, and the hardest to\n", + "get good results from. If your character sounds noisy or robotic,\n", + "try improving the dataset, or adjusting the epochs and learning rate.\n", + "\n", + "If CUDA runs out of memory, try the following:\n", + "* Click on Kernel -> Restart, re-run step 1, and try again.\n", + "* If that doesn't help, reduce the batch size (default 32)." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "epochs = 200\n", + "batch_size = 32\n", + "\n", + "# Advanced settings. You can probably leave these at their defaults (1e-3, 3e-6, empty, checked).\n", + "learning_rate = 1e-3\n", + "min_learning_rate = 3e-6\n", + "pretrained_path = \"\"\n", + "load_checkpoints = True\n", + "\n", + "import os\n", + "from hydra.experimental import compose, initialize\n", + "from hydra.core.global_hydra import GlobalHydra\n", + "from omegaconf import OmegaConf\n", + "import pytorch_lightning as pl\n", + "from nemo.collections.common.callbacks import LogEpochTimeCallback\n", + "from nemo.collections.tts.models import TalkNetSpectModel\n", + "from nemo.core.config import hydra_runner\n", + "from nemo.utils.exp_manager import exp_manager\n", + "\n", + "def train(cfg):\n", + " cfg.sample_rate = 22050\n", + " cfg.train_dataset = os.path.join(output_dir, \"trainfiles.json\")\n", + " cfg.validation_datasets = os.path.join(output_dir, \"valfiles.json\")\n", + " cfg.durs_file = os.path.join(output_dir, \"durations.pt\")\n", + " cfg.f0_file = os.path.join(output_dir, \"f0s.pt\")\n", + " cfg.trainer.accelerator = \"dp\"\n", + " cfg.trainer.max_epochs = epochs\n", + " cfg.trainer.check_val_every_n_epoch = 5\n", + " cfg.model.train_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.validation_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.optim.lr = learning_rate\n", + " cfg.model.optim.sched.min_lr = min_learning_rate\n", + " cfg.exp_manager.exp_dir = output_dir\n", + "\n", + " # Find checkpoints\n", + " ckpt_path = \"\"\n", + " if load_checkpoints:\n", + " path0 = os.path.join(output_dir, \"TalkNetSpect\")\n", + " if os.path.exists(path0):\n", + " path1 = sorted(os.listdir(path0))\n", + " for i in range(len(path1)):\n", + " path2 = os.path.join(path0, path1[-(1+i)], \"checkpoints\")\n", + " if os.path.exists(path2):\n", + " match = [x for x in os.listdir(path2) if \"last.ckpt\" in x]\n", + " if len(match) > 0:\n", + " ckpt_path = os.path.join(path2, match[0])\n", + " print(\"Resuming training from \" + match[0])\n", + " break\n", + " \n", + " if ckpt_path != \"\":\n", + " trainer = pl.Trainer(**cfg.trainer, resume_from_checkpoint = ckpt_path)\n", + " model = TalkNetSpectModel(cfg=cfg.model, trainer=trainer)\n", + " else:\n", + " if pretrained_path != \"\":\n", + " warmstart_path = pretrained_path\n", + " else:\n", + " warmstart_path = os.path.join(cwd, \"talknet_spect.nemo\")\n", + " trainer = pl.Trainer(**cfg.trainer)\n", + " model = TalkNetSpectModel.restore_from(warmstart_path, override_config_path=cfg)\n", + " model.set_trainer(trainer)\n", + " model.setup_training_data(cfg.model.train_ds)\n", + " model.setup_validation_data(cfg.model.validation_ds)\n", + " model.setup_optimization(cfg.model.optim)\n", + " print(\"Warm-starting from \" + warmstart_path)\n", + " exp_manager(trainer, cfg.get('exp_manager', None))\n", + " trainer.callbacks.extend([pl.callbacks.LearningRateMonitor(), LogEpochTimeCallback()]) # noqa\n", + " trainer.fit(model)\n", + "\n", + "os.chdir(cwd)\n", + "GlobalHydra().clear()\n", + "initialize(config_path=\"conf\")\n", + "cfg = compose(config_name=\"talknet-spect\")\n", + "train(cfg)" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "N9hh4WPHbCcn" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 7:** Generate GTA spectrograms. This will help HiFi-GAN learn what your TalkNet model sounds like.\n", + "\n", + "If this step fails, make sure you've finished training the spectrogram generator." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "import sys\n", + "import os\n", + "import torch\n", + "import numpy as np\n", + "from tqdm import tqdm\n", + "from nemo.collections.tts.models import TalkNetSpectModel\n", + "import shutil\n", + "\n", + "def fix_paths(inpath):\n", + " output = \"\"\n", + " with open(inpath, \"r\", encoding=\"utf8\") as f:\n", + " for l in f.readlines():\n", + " if l[:5].lower() != \"wavs/\":\n", + " output += \"wavs/\" + l\n", + " else:\n", + " output += l\n", + " with open(inpath, \"w\", encoding=\"utf8\") as w:\n", + " w.write(output)\n", + "\n", + "shutil.copyfile(train_filelist, os.path.join(cwd, \"hifi-gan\", \"training.txt\"))\n", + "shutil.copyfile(val_filelist, os.path.join(cwd, \"hifi-gan\", \"validation.txt\"))\n", + "fix_paths(os.path.join(cwd, \"hifi-gan\", \"training.txt\"))\n", + "fix_paths(os.path.join(cwd, \"hifi-gan\", \"validation.txt\"))\n", + "fix_paths(os.path.join(output_dir, \"allfiles.txt\"))\n", + "\n", + "os.chdir(cwd)\n", + "outdir = os.path.join(output_dir, \"_training_data\", \"wavs\")\n", + "if not os.path.exists(outdir):\n", + " os.mkdir(outdir)\n", + "\n", + "model_path = \"\"\n", + "path0 = os.path.join(output_dir, \"TalkNetSpect\")\n", + "if os.path.exists(path0):\n", + " path1 = sorted(os.listdir(path0))\n", + " for i in range(len(path1)):\n", + " path2 = os.path.join(path0, path1[-(1+i)], \"checkpoints\")\n", + " if os.path.exists(path2):\n", + " match = [x for x in os.listdir(path2) if \"TalkNetSpect.nemo\" in x]\n", + " if len(match) > 0:\n", + " model_path = os.path.join(path2, match[0])\n", + " break\n", + "assert model_path != \"\", \"TalkNetSpect.nemo not found\"\n", + "\n", + "dur_path = os.path.join(output_dir, \"durations.pt\")\n", + "f0_path = os.path.join(output_dir, \"f0s.pt\")\n", + "\n", + "model = TalkNetSpectModel.restore_from(model_path)\n", + "model.eval()\n", + "with open(os.path.join(output_dir, \"allfiles.txt\"), \"r\", encoding=\"utf-8\") as f:\n", + " dataset = f.readlines()\n", + "durs = torch.load(dur_path)\n", + "f0s = torch.load(f0_path)\n", + "\n", + "for x in tqdm(dataset):\n", + " x_name = os.path.splitext(os.path.basename(x.split(\"|\")[0].strip()))[0]\n", + " x_tokens = model.parse(text=x.split(\"|\")[1].strip())\n", + " x_durs = (\n", + " torch.stack(\n", + " (\n", + " durs[x_name][\"blanks\"],\n", + " torch.cat((durs[x_name][\"tokens\"], torch.zeros(1).int())),\n", + " ),\n", + " dim=1,\n", + " )\n", + " .view(-1)[:-1]\n", + " .view(1, -1)\n", + " .to(\"cuda:0\")\n", + " )\n", + " x_f0s = f0s[x_name].view(1, -1).to(\"cuda:0\")\n", + " x_spect = model.force_spectrogram(tokens=x_tokens, durs=x_durs, f0=x_f0s)\n", + " rel_path = os.path.splitext(x.split(\"|\")[0].strip())[0][5:]\n", + " abs_dir = os.path.join(outdir, os.path.dirname(rel_path))\n", + " if abs_dir != \"\" and not os.path.exists(abs_dir):\n", + " os.makedirs(abs_dir, exist_ok=True)\n", + " np.save(os.path.join(outdir, rel_path + \".npy\"), x_spect.detach().cpu().numpy())\n" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "Ajfyfz2p9Ior" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 8:** Train HiFi-GAN. 2,000+ steps are recommended.\n", + "Stop this cell to finish training the model.\n", + "\n", + "* Click on Kernel -> Restart, re-run step 1, and try again.\n", + "If this step still fails to start, make sure step 7 finished successfully.\n", + "\n", + "Note: If the training process starts at step 2500000, delete the HiFiGAN folder and try again." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "import gdown\n", + "d = 'https://drive.google.com/uc?id='\n", + "\n", + "os.chdir(os.path.join(cwd, \"hifi-gan\"))\n", + "\n", + "if not os.path.exists(os.path.join(output_dir, \"HiFiGAN\")):\n", + " os.makedirs(os.path.join(output_dir, \"HiFiGAN\"))\n", + "if not os.path.exists(os.path.join(output_dir, \"HiFiGAN\", \"do_00000000\")):\n", + " print(\"Downloading universal model...\")\n", + " gdown.download(d+\"1qpgI41wNXFcH-iKq1Y42JlBC9j0je8PW\", os.path.join(output_dir, \"HiFiGAN\", \"g_00000000\"), quiet=False)\n", + " gdown.download(d+\"1O63eHZR9t1haCdRHQcEgMfMNxiOciSru\", os.path.join(output_dir, \"HiFiGAN\", \"do_00000000\"), quiet=False)\n", + " start_from_universal = \"--warm_start True \"\n", + "else:\n", + " start_from_universal = \"\"\n", + "\n", + "hifi_train = os.path.join(cwd, \"hifi-gan\", \"training.txt\")\n", + "hifi_val = os.path.join(cwd, \"hifi-gan\", \"validation.txt\")\n", + "hifi_wavs = os.path.join(output_dir, \"_training_data\")\n", + "\n", + "!python train.py --fine_tuning True --config config_v1b.json \\\n", + "{start_from_universal} \\\n", + "--checkpoint_interval 250 --checkpoint_path \"{os.path.join(output_dir, 'HiFiGAN')}\" \\\n", + "--input_training_file \"{hifi_train}\" \\\n", + "--input_validation_file \"{hifi_val}\" \\\n", + "--input_wavs_dir \"{hifi_wavs}\"\n" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "yVBjGhRB9hUJ" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 9:** Package the models. They'll be saved to the output directory as [character_name]_TalkNet.zip.\n", + "\n", + "When done, upload it to Google Drive, with permissions set to \"Anyone with the link\". \n", + "You can then use it with TalkNet by selecting \"Custom model\" as your character. \n", + "\n", + "This cell will also delete the training checkpoints and logs.\n", + "That should free up roughly 2 GB of space.\n", + "If you wish to keep them, set delete_checkpoints to False." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "character_name = \"Character\"\n", + "delete_checkpoints = True\n", + "\n", + "import shutil\n", + "from zipfile import ZipFile\n", + "\n", + "def find_talknet(model_dir):\n", + " ckpt_path = \"\"\n", + " path0 = os.path.join(output_dir, model_dir)\n", + " if os.path.exists(path0):\n", + " path1 = sorted(os.listdir(path0))\n", + " for i in range(len(path1)):\n", + " path2 = os.path.join(path0, path1[-(1+i)], \"checkpoints\")\n", + " if os.path.exists(path2):\n", + " match = [x for x in os.listdir(path2) if \".nemo\" in x]\n", + " if len(match) > 0:\n", + " ckpt_path = os.path.join(path2, match[0])\n", + " break\n", + " assert ckpt_path != \"\", \"Couldn't find \" + model_dir\n", + " return ckpt_path\n", + "\n", + "durs_path = find_talknet(\"TalkNetDurs\")\n", + "pitch_path = find_talknet(\"TalkNetPitch\")\n", + "spect_path = find_talknet(\"TalkNetSpect\")\n", + "assert os.path.exists(os.path.join(output_dir, \"HiFiGAN\", \"g_00000000\")), \"Couldn't find HiFi-GAN\"\n", + "\n", + "zip = ZipFile(os.path.join(output_dir, character_name + \"_TalkNet.zip\"), 'w')\n", + "zip.write(durs_path, \"TalkNetDurs.nemo\")\n", + "zip.write(pitch_path, \"TalkNetPitch.nemo\")\n", + "zip.write(spect_path, \"TalkNetSpect.nemo\")\n", + "zip.write(os.path.join(output_dir, \"HiFiGAN\", \"g_00000000\"), \"hifiganmodel\")\n", + "zip.write(os.path.join(output_dir, \"HiFiGAN\", \"config.json\"), \"config.json\")\n", + "zip.write(os.path.join(output_dir, \"f0_info.json\"), \"f0_info.json\")\n", + "zip.close()\n", + "print(\"Archived model to \" + os.path.join(output_dir, character_name + \"_TalkNet.zip\"))\n", + "\n", + "if delete_checkpoints:\n", + " shutil.rmtree((os.path.join(output_dir, \"TalkNetDurs\")))\n", + " shutil.rmtree((os.path.join(output_dir, \"TalkNetPitch\")))\n", + " shutil.rmtree((os.path.join(output_dir, \"TalkNetSpect\")))\n", + " shutil.rmtree((os.path.join(output_dir, \"HiFiGAN\")))\n" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "5OtwfrTT-blU" + } + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "TalkNet_Training_Offline.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/models/tacotron2/ControllableTalkNet/assets/Equestria.woff b/models/tacotron2/ControllableTalkNet/assets/Equestria.woff new file mode 100644 index 0000000000000000000000000000000000000000..6234e41d0c11824c00017443f9b0f635c0b7af4c GIT binary patch literal 22008 zcmZ5mb8seKu>E4&wr$(l*tTuk&c^n}ww-)2Hg-1Nm>c8e_uhZ6t4`fHr*7Zw>6*DU zGd-S)l9B)rz_+PC1Cah3YlXi#?0-l9-y|i~t#2LwKiDzrtVoGVNPcT>-<;+f z^q|Bbpi)YTs^41SHz)oNc1doDQmX2r-`c=8H~mJ~n!w7fqAC+B002t<&DFnQzhp#UHn5CV!1pol?O=UO%fSfIGTP=Jm zb7M0AK!N*PWcvmuvFGNS)wlSq*?n{3Z;&HiLp)pAyLo+Uao;h(zhgAmV1h5%IhuaY zQ~LRxO#{c@?f*?5<+naxDVc&CFzkP>EwbG)U99_T9Uyc7; z>-=6*tZ|e3zZ!vj128R6Jon!?_o93Gf1@6bSFWj%nUT>SFcS<&WFr$J3nQ}%$bH-h zUgG-vJ&vC!k)^?tEEC>k&0%yy;bWnNN{h>om!G;13_Y=ia z^iISxNxa(K`jI5*aDqJd(!lMY11}CFH(PsijFI+XUv!R z7s>zbtKZP{*?d~)OSYF?rx|*9REo%C@vlF2AAO$Jv^>sto!iRVo#w#`0t6=DVrLJ= zEhc1LP+;e4m|}~dqCS)%3Ps@%0OKxAy4OL1iIj<}#6z9LW3AA@OypptSt{q%77n@7 zQ}%!H@F*my?#XeVD|^IY#0I{pkszy#re6M%6;eNpW>qFLQ~6yk`-v{`SZ~5H8ONe# zJTkmbB8<=Ek(HT=UGU7qc&Pg<*%w@YV*8A0FkXK9+Cx~+x5lJ$N|w-+(h0qW^BfM#$5RDT}8 zT6M~ESRev{7Do<22~i3o!AFS_M=wSR)fyook9gHS@tpa~jNHP)e9L$3U}62=+MCQ%iD3ll@qe4mw}E`$L!e%{N4+q)=*z9lJD!!L z)z#fq)m>d(zCJ!#I~Kd$zDArzm~FwvImQ_4BA625Vah0K$mkeOFf6c;(Uh6Q*&Y1x z$D`D9;w+Ska&g%sBjwt0+U(gCOYhgUS2z}hkqn=GOuJvdr{y#?)mM2=@_ip&>mhW; zg^uNqbg_o%&zV6dDNJmsGuwr+>u3jCo0ALlK1dB6uq_(|R- z;C6<+-o*|VBk3{M-q2zp;s$*gUeYOJAxdqk77wL7Z5g5Fa}kkg{=lwDixlHis<{l+ zLNy$dTJk_0YWdLSQ$tN`sD*ouM4@=+Cjh9aoeobOwl%EW`FKV)%VlSKa1o;DBLOW{ z1~P6eyl9P3u(mfEP_bZ9X_DJ%vP;_m^~l69WMcWKQ%as&bLh+XY5|W4DakZH+DZc1Ty->{$`A*V@SB)wdNT1_-PnC#OH)B#DYc;ktae?dPb$<(4pCa9aUe{j`X@T5@26AxNDOhTjF zul3}WINHGU(&YS7mAOBMO|E4B2ood~VU+#n{1g1aZ1O#I$2LG(=y>6iU@(l z47s7m`Gq!gEtVv)rlea;faYHDUj*bzbR0n`M`yU>0QjOihWl%WmgN8$M24sXYB02qJ z0b|sKdagse?C5ag^ewijwrS=iKOFP00s{O!_c;Cl>y3s(ds>1de7@`$j23C{)ZSR! zoQnXDGzCmFsu{d2Aj>)kwV!hvoSI@WETGcoXDL+?Q4v3nsluR3ZXwwT;DtDd+Wu7H zc)B#yD#42MN8lZ)CRebJaAra7xSujCrZDrO`t_0?Y#d)Ku7vLZ#?sI%%m7O_K6-Oz zIr$xm&oA>sG_~@>LtV*KNr3=O{ZVO6cT#*pNNkW|^-m)lzhovk^}ouPu7rtKCQ^3i z21B$l`)1tg9(>F@@IdLu>P>P0NqtL3I(qXAO<*P0poFN?omvLHj|o|k7zzlUg+0c8 zS#^V$b5}>$lmVLXRRxvVJ%nFr%CRVj>|of>9{vHot0?qXj6Img@xLL;qzCllNRsG$ zj*I$?iB=i#2hK2;U6Z0-) zu~GHqC75pDbGx(={^dBEHfHj0k*4J-#oV3DxVGZ7f1QhM+S%G>N0hR}o^0 z;dSX0;0G;29=-)N{1vR?IDcYJ%lF>2pDu1j!6urNX65bUMUNRbnO`BDi4h!zi)qgb zrJ(_fQS`HpZ5+qBqZWFilJ$rVc?BVM6pUp84F!TPlv@&2z0^;uXmwc4{h4C<5}D=% zyt**ILL$~o!dV89?gW-h1e_)DZ_-#i6c@3CCd3e-!uWmWB0&`%??;F_=u-+Cqlh!@ zA^aF%g^b4G54=__;(;%}XpBh;`||k>CghRd_I2Y)G^%au?IWl9}VM;>=O2uYR)1T3VvAt01)n z@eAaDGs&{WQ)fD3iQkg#Ljc}(e4zYu#Y$7sO%e^?X(}wJD`av}F_{z;cfN=Dp&2h9 z+}j8}@f~w|)BpF-uuaA;hKSU9ILS=Z_IL(jaR)F(Bd0m zm?JXyW8T&HpK4tDoG_j`!q<+<_;JJ6x;kvQJHKiPgVe-$utlL&$9VurnR>Krp@8L+ zEN!bvzMsD(PM4M$YmLkZ|3v4aGQXhY*8P<2s2(7hXu2A|UjsGc?YQuu3JM{P_Q-G7 z_MW6wkXXI#w(G-Mzlc)KTOsX_5y<~3M9mxRMG_PCeviykGN~dgJ~BWe3(MA{jvQ(e z5RiYIZbo{ZwL>I4jj00)j>Vc;wYk)SQ)}ZABRIR?|KH@xqZ#UE7*jbJXX8C!s%vx9a(u}xB z%|M>=jA+c9ZiO=e6k>2RJQFRNBC~HMhM5UWE9ysRQ`9A=Uhg`DJt6~9G5;!>S6IJz z2J*am|MpCfg3?silmiAFNb6nHIQ&-HLLL__KAdT{Vo6mz2jhj|A=6Ea;Pw{{Qk zWEPL5IeZv?54vEHFQ8km1Ku`RQqC)^)x&iFpDp&N+lf)f7Doz^TBHSDQCtqy>Z2?( z+6D7NoM9PUn?_6`><#^F=rotq$U>22zK9sU=I+Y7|*3PB4*|fv@a#<@Uya! zm_8q%M{&V}W9RCDJkqS8EK4tF#R`lxzSPJcrf>7n{T&KY%??yG8CEh0o1|*QyQ=|= z(ZloT11jN1OWX12Ap}V=n(sLV%ucGnGJ+`66=q%Tx?y(A6{M(L_!oun^h=Y5_)BH( zCD{{)W&2caZ5*_3$g6@UA$qiq&6SU1?l6G4{+yb9X5k8&^GB*2gv#ANGsagBk(iKF zwFb7G5VkC$Rp8dh;r4h;utaEC(X^vyl&{!d?oWD{O|BW1M1rG4qF4l-y5PiV)OI+- zZ2;XwvyfeIdzQ%RQlQE`19e(b>R6Ofmoj50j{%T6gIYf-h&=7~6UcCOVU$|xovimT znOM$XP$0_Z@LXI11h1l($AeYohama2$RN{Om&oXXk+a}D?B$dT`)Fm&_R4z;NHD&_ zCq-G8f+huROwhI3=bVQD6V*9&_xwSz$lXO=%!y+Q&#ed4i;HH8)C@eA`0xRG2RkihK(*4 z&%{mjzHnZbii0wgVn+ovFfv3jcd}$3B^CFl31(U%a0xywd3X$nySw51k#7z^NX>_z zO#{Um01+-kgXS=Y-Fk%vT5>J14V9Uct39ZS;2>fa(#&uV7n(SQg2<2x4%j)!+wKM) zayW@If|Uzs71D%z>0Ckx@(BsOqM<{`j$km{!=F19?}`nq1ojq3KG-|2_oV(J7eWLFyFDqwfd*lET8&(B|N+hY27GGTc9B zC_OAR%rvKBePNL>Jq#OXrw9xqhG47*Nt3%B`~W?-Zy+TJlo9Rfa!tWcyI_{?`2lVa z_}7|0??m4tAtjeWs7s*tT;}h|xSR>>SWPn&M#2}&`_w3U#6d6LZ`z*m1&2HMsc=;= zGe=+v92&B)YlL=F=s>n!+H_`)e}*MT)K`n%L$Q8r1W|r+r9)nEty6;S_<E4>FF;#8O~-CaN{;!+=M3z^v%_S3zvHNV5pUj zy4xr)Q1;a)=Nqs4cmow@$MQMJ`ik5sDUSFsHTDXFJSKHDM_hCLJnE`i`YORtVdv!a_>`SD`6588nYImF7kp?5iw!QwiIDC8WTEym#QG3nANCE z-)Ntn%O!Gg2$U(_#9;`8K)oRU{EpuQsW@iO!GH$20di=VIBj@$3#e>NdZXSakxK7r zK68m=UAiBX#=|G!*x_#!qg{s_)&6To!7-W5N&GW+<1m#bo)A&}M%iPHlxrf524fu) z9Ad_g!vf)5P;P58nr(yTPx~C*49rqSyk}RgOQn!g^)IAx8N&YIEh2sF?in|Jnt)Cf z3zG|LJi?_wh50+K(xIHbInzkW2~gyR*ucDPpz<<5NwdfPf=-O7Ghw8c?FTZE`TWMY zrGz>D)-w4aG#fvZg-AOiMY>}l-RdZ-&>glm&WpEOqUQb0u$37hKF*}RC zAyze_JWToP2G3K`2?EeDen~6)3tG3>%epl0ui5+?*-~)1))_q}YbnftHq6Ocb0w>( z;(T;geg;G%PIatJ`ZQhZ9`5`hShGQ&z$r0oyjLQK@`E%dldcOLIcI^gx{5*mK}8FB7=jtKt&R&;)1kb&q`P>&JBVbI5l{3*+3 zN^n~gc~da|##MaYt_#dgehU-e6DByt*~~D-xpxD$&uqL@ei~2iB+n`li;!;u5U2IP zN(AUQ8Hku@CYIvP(%mK+{!V5&=c|Xf11$xwS9w8D$=D65;XoqOhW9qsNAVAmjBWE3 z#p!>)aw5S4W9~x@HMm9kLrxe0z{9EXxJ>~&w>NGsWiYK=eE-*#PD89B$I+*Z;onLJ zULy(!vm^#jW(<%GQ##f04~eU2=3Di2>2Fwk`YQ``a3x2haP|XnbJ=bCpk`!k`{Rq+ zu_u;NhXlkgM~v>xO?nB9GpjxVjQ% z1%9@ATCqoxA&M2y7uMTdl!E?3(>ADCQ9VVgKLbF6t}Oj{fIv49x^r)LM8Uz-gVr|b zFWQY&>6_VO%E;q4<^oI|zz%&1L!g{t-!!RRd9JC|Kl>JYYGWEeYe{ao_!2k|8-;OG z%2fzTOhX<6OmLz+HhHL~WF6!Xp?{OtrtL+6AVCEFpAR6swosO%rjs+ z$JfGflQW6GIL1*+tTTouvc#l22*s$Qhk44>Q;_9t8;+5AdEOkv5$8czJ*d@*OrTGv zSuQ^C1%m=Dv>iJ`H-eAZ?S|sT7tM{i5PV5(529VPK&__Zq9y@BW!G>cG=9-~b^9eCRtFaJr+7LLl{DlU?BWx-|?ek>1ZRfzEau>;9p;}p5 zXP3LFZPENi23dU7@>%!O9a=1O98w$r8XlpC)vPcM9NmDN5!lb7J}N7ap)gcZsN$0P zN1l#Mx_VI~=ptsv6po2{pjh#Yjr+^{aw_}?xn&TYp2pr39M@_`wvD+)YYhC#hivT} zMm)IQO2^+QeP)riv+q|*B>G$Rd%5;y3Nk+1_82>Htg;`O&E>hkh!uq^)cerRrrCD$#>|4nLdMO{kDPntX@BD#H|F?u@NZS`{gq%< z!Ouhk>!7Q(v$C;oRH1_xB`(aAX{45}Z0Gb|2%*~U47*-~u*P{A56j0NuHFX|Ny^#Y z-P!7wN>$q+&3$!Y)XM>rqg(VvDWL0ehKmm;urbg05Er`#D@ZZl>sf^7u+sNvye#Nx zA-q%qqfiIc9)U{|ZiVqINCkZl0xLaDL2d25ilW5AuPZgL&%dAyr`_VAth5~;E^ZUG z4P6u>MZdBNJKD8KO zWHrUj1bMEEOu!`~V!zt6@TF3m0;~Q<*0&dI?bGw#^-4$}blz3nE0XO@#AV&pv%h#0 z|7u=e3|YHwzvQmAL(@8{8z1S(a6)u)re|?+s!Gb3|ErNosl=A`-eNf{_?F3 zn>1&X!0TS4Wjp=ALMziMox#mX+=H>Jo(Zp~-*>DDLeKWdke6TX?*!Z21&Z}h1dw?C1&NNVZ{HV_r1}C7b+m8w8lTtiH`-M^A_LIXP8PmZd1J zGh;P@@IV?UgbW!Ci&om7^_d3#xVgp91zu3^46B7~J`bh410xsX;xAD<2>S)?vmM?| z9`Xf(4W{U%5}ocfZo5WYf&Q+9#S6WN&)4H4)xPZqY14vdsv!Dgp#!H1+N-3gFuBww zl!NSyC}sRfvXdk1Q7C0WW{1>4F2%%2atE?kBniHed;wv*WVPOQiozSFTa*lre4v8V zC(phI?~MZ`4LesjmeEmE68cxk-4*gth7 zlri{0ZM8sNbQ%;Vo__RrA`pgi=~O{W`gG`6a^Ut= z4dSwJH0ASfJ=-x{1eBApo`u4F19exbw=kdLioW8tfa}8=HPZfBH(XQ5mKw%1k z=ZYvaWx72JY|PTP9;(ttR%p1`w-!swPoPoo)RM(~VeZ;toG4b@k`zmJ-57O_Bf>zjGw zVQr8%*P54sc_z+*FHm(rdGH79&$ZbZ!E4h!!a)L^Hx2f+Pg2Lr+zYMjJ#22PM2c|f z?G6_o*e5enE!6qIaZX4sj_n6;{$xvbHjL^TJ1rb`akF9srM!~T1G_(54v7Jp)|%Bt z;yFQL9)L^+q)+x>=P*m7=x{Hq+mQdWU7JAVptn8<0qDD%jr|bWF+J=FK8`P7^}h51 zkTaY;*A}q$1{Vz1CN2Lcd-$c1lP^2W$HJ3n+ZWcc`i zR9O9C=nJpc`h_$`^mD<};J{xn6zt*Hy;*YC}P z!rxAI6&ssH9b{z6=`h)W5*ee;KIizA9=|tE?XIRJuV1$@^-f!9_YOAXRoBwYolHmh zf{MGU+~(V(&!N$~I2K!hU#phwGBery99hZPj=#C|fmmQrmPi?&tMNGl(@E{|e4l-{ z&E{!aY_ItxW67@`XRCNvEH82;rAx`bT8iA&b9Kp@vLdQ%ZRK(pRDy_uqo>sse*Ao? zM6HZZ7GC5CQRLqbyWin$@(VZ6tz(Kg1(xU5ft_vBw(1!dEZL&Y8ew5)VKgO@m!kqRB!rXoaC&G7+yhU8bHHYiF`s zGXS@!WfnB}`KSMWq@~nIuwMzEVCyE~){=N68&sh6naCW;Znvg3PO!(y?~iA{LdaEP zT9=zKix7see~ZBDkJE|ne_oOWeqnGpvDV!`E9KwMp`|XMxt6KTbf_vk$U-;ahl; z{wS!-UVcFNgpOD7-_b`*D`As_u$__Lu}GRltB6SkDg1~UnDf0zJ2Kob^~0|-MBK@P z+K#`~UpVG&R`XPRojRFgAVBGX>_PCbOSnfOPh>hgg6RH};fSQP@!9i4^vw29G-@e0 zJcOc40Rocc$#IC29V;!#dXIWzI*!ehiqpR@rSOGxP%lFm1D6Q=*fk6lO!YeG+)V7> z^}*(c%>0>0-F2E$hTAfrz%O702E01vfx`-m62I_KSWtbfn-HgoeIPi}qx9eQ3_g z2bTQH@bJ!gT|J5?Ww=GlDHfz6Zl5S;&k(0501Pr@~R3asOyR)Wk{(l$YCax zZt)gA>@v@|NIOUIJ3;~tdii#s&0X-BW^28oCmG0tCx%gNy{I;+Y??)VilF3duknc%gOnBPX?`$foC7fa+!4>j z*V~dO*@)!0A7gP?8g^chCn0cc_dYL=CB8RrsNRb=P^Q1de0AJ4%gHaj{~+GfR!hmY zU`D{nRq6PF{ajtS;Jn-u+VB6f$@U4EmGR)rA*W->vv2;8H8=v9poB|lvzTu2Gs*}? zJb9AqdHK{+$NSSe8hg<1x4_u$E_|0}N9n=)y?v@p7vnZuh87Qvwc3cqxq{yt`wsa+ z8W;ZV1P3Ei_l38K-S0r3J)4f|4!O2?{qfL9dF{Ko9bR-3B<;EqZ-tny8cj5)R+!S9 zOgFUIj;3`pxFimUEGrUsiozagzkH5(;8?Vla?=$y9|;ihxGFxEJuqT&&|l zC;-yokav^V&FDDH7sbosz=-ua-p?SGVdugqR){~)}e15c&-XorA_ zhL+fLmQot^sI6%v)LPzekDy~HvQrzV;rJ;hb?HCQ1P)0pZ#8fGUo@b_J}2TtK+_Bgq3noTT0!89M)c4ZOB zLP(jnc+CnipV19^5X7YDsFFtu>*xdANuz(qGgP-f7+O)xngN@^tb6-%KSOzMvP3h^tPsqpVrYMy z&sa-z&GP8CAjbU}Q*kK#DoOI9{F}y5R#S;c+)RqE>LJTxjSt`Br|~Yzy+{8X?zFsq z1Gg_90HS^ZWE$&d@qWB@%*E>usuB#~ndOIiYJy@Y9u3JB&UOODst3?VI%M&=F59~v zyKm_=lqd1H7>_lS!A;%Um``hDxxRGV>OL6M8`Ds+mgQnswT8t6&5p{g)3LO6$o*~4 zePKCMMuN~J}RZ7IqsLb00ma@#zgYlipTX(LhyWf``(q3D=1;F99?8K!40Rgw?N0=JtlpXwH1Dsr z@=rf;2E4jM9Hs}<1f73PWv8nl{6+IIDKg-g))X*%!$7WW57+2gkU2ti|Be+us|E zSDa;Xod4Kbwg4G>)}bbnBRcgHIBeO&O@xn=+CqYD_Ox`8@rfQg`__6*bE>RDape#T zK+EbKHiOL~y`FZIsQc_7jqbK!g!!Ika|%DW48$0$PS)FENhg;cX|QwMcKRqBpP-TwpzF4!n(fKYOG#u(yQX1AT7G+tXc z`x48c`x2GpjgK~&=zb{-PSjKN+}Ns?j1MtLhkVPe*Y33Yw=?jaEh*cL>SlYH=!BMA za}Im_IVO|u7`upZ1j}m`O{cSebHR#-nmqoUID(x22$mHzHPvMd*7s7U)cA!IgJGSe zm2@s%6QyDETT3~1p!p*6N&=terAE)lAH4Pq%p||5K zCSUhnJYhJr`H+be;QgOt^>&Sk;ATUOnEm0pMXog0fw&!|!l59A0{WgPwK5)9MtaY6 z?FIT`NuBr28wuxpuix`o2q8WcgB`W^-keSx*17o4p=uqSB_YJ~)7zehfnfjXuyP>m(M7#C~X)@{*<(k+RHk{|Spi3juaL5=d51G44y)oqjjFvmpG zhpB@G*pvghB{E0*n=PV|WozTQuF8fbz|yZzuEu1vHGRi)?bc8cTwA$??exJsF}O*r z_|WppB|Hx%s_;DoHEHHy{C(lh(+Q*R`r)=nno{rqqp`fBw(k~I;H>FEvYYWZU!ctt zB=oww#`m;OABeE#JihEc7*9LZv>!&5e{qxPj577lim-7Nax5~co5jozIv#dyRju)& z5%Zupc5K;|j?Q%SbV&C?Kxm-7xyJFA1AX31lOmxqJrJ$PY%56~x4pFwsoeR#D*^`fm*eIH=NDRWona*TgdJJ%N<}A7qm@!&w zS12ogKwa&3e#Y~pb}G24Dv+TMB&5!(nC5{8rn_x>XvNX~82B|y(0|ip0z?9&6F$>^ z9tAxk8%O8f77ZKOZX0s~1(C0uH}Zh5^@$jomydr8eb7BN=qRRd6HR=fk&8)VeA_~_ zjV8#0xlTUhy*OXbPff&9&tFZ&%yy7sfTs0qp=m&Q%q?@7ljF?oZ zU5Dzq(&RI&j{_hOAIr~1aBDIaoAR0)NO=8v1JOIe=l){}wqe-;QqEQ*QjWW87tSZn z`}|6vHcYpy$9i^B=Qqtf2clav&Lp0cjNk^KPt%WX8aVxM74h8)K}vGL`sB@#+VAv+ zPE&uUp75H-*kjUiuAq(_cb5*`-`1v^HGRih-v87m)c$$ttjV5udNqRZafr%P+K7WG zWn&>pv>GTUy}Q|--F2T|b>4s+FmQGhIs;1? z7z`RPxF<0ly9LArM@KN3eLZ~eD)s_;E`mrYnKr}@Cetr(M0z-+_@nqZ2+cN*wMiu9 zLbvst)Gd!jjSJlgatv_KL>HpK1iyk5 zfD03M6Psf(l)zeJ1))_BZrxu1e3ll+JYgs{v@&OK ztKb>Dcz#2&1=}&id%q#NY}Y<3Brw+SJP|BwYZnVjHWwdC_BY)FZYj>gXs{BWlLKl& zOzR3xyL03E1%CalavJuVaghHPjx%X}Om~&UQ)|4OEpERBRzbkq0o~QbF(?B)A5+U! zdAs25P!}zKci^+35G@Lr>}U?(K-H6Y56wCAGXdmLG3#+c9nhyCQaED>aKNg9JmAeR*Z2RCa;Y6az6;O0{5ruj`A` z)g3E0Eo<=t&}K9Zg(StjZU`QhL_d3ki^Q$ z{2(J;SBkf{l@?{2=I^w2ngmeoBvZ2l=Qj^AUU|04(ELa1+k z-6}Ouj|h8&ANI8mkPhoBjIk>`s=Y*;Gz4r#&FC+oWdH7^L>Q{OG;*UOVJhhA zpYk#lb(R1HSej{$r*s zEBGvQ|y!Jj@NCoYS&)?IzimSqQmd^rb!SRY{h3bk|;>9amR*JVSaJT_3_ zH_A@PH7@81iyNQ87EA0pj|Xc-5RsH)8t0s~&r_$62^(XttA#sKrt}kTZ%^7lm2}k5 zCl4XmULXdWGBxZTE!ovgt}PE+_)Pa*u5zs-y+TTXj)==C(L!YhU}(r{M(UxOZ+9g` zsejoCoc{u*z&o3c&r(!THL8iHJ$P>mTf|9Y<}cE_uf{DXL9ypx)CQ(lv(y^`R4?!R z`By)GM2f4{qv2XE=jYZ|Fb-)!c9|*xi>q*p+~?BmGyfTB8*UW$g^tTi1@(-&u;Oif z(9#K1n59g&dYRl%Sr84-oW_|IJmP7A;VN2HibxbWt6|DpE)c9Pcli68=Rs-j=)@{L zDvdjy;pW0jn8`lwELdMMA?Qom#e<|3FKc4_wTLmh5}p2+;lCkrj|zB2+Iyc;l+VAh zNIpV2&_}Vbm160F2>O&&v6@WC@YTNAy#OH%MO{kaZ%PWm^fPO*4$ zYbEXtuPCVvt$tfiD|X}L3QNhTuzh)jI!`(MX0I>K3k|Q2+bZK-=!vZNYU%3EZwGl# z$Tkf~hdTr$;?pKehmJj**XF+kkw{-DpAXBam3v zyJCf#&JPu2LibSbuxcZ5iSdc1g~iv~Snx=cOP`E>N%Q$sgjxibKhBtfm1cHGtDd*D z9oJ{fG5N>2$Hgbu7XF0*7*TS*_NRmDwJi?Jy2{V;{hLwiN(U$+R8g|i|F%MeRz~XMgQe-j1x#W;6jp-VVR>; zGHjI9&6z%C?pLT7_hBI~L(-U*OeKZ1@R0;zAG9OQFVImnlE=Wco~2AEb2HhdD*NN1 zZz#p@90sB(e9dO$LBi0yPt=lQo^*D4N-v=r(*IvJ&66VE!~Pthao@t{X{g+4InvPX zjbh+w{ulXE_`u>i;jcyM^TpQl+&YlRjh{ehuht_4T>ZqXw4~^YPV|wWajvw+_MXpaq_6LdH7x%uPUd!VW`nOw9Zo8b`s~=fTPvG$MiT<}ux3TYaqvEqef;>41 z9UFHFD7~PxfANA=0n&>RsIse+gq16cDAAVrlvcK8T8` zj9vzge!}S&f9U=F$76_Y@qt+2;0WE_D^co=t8|A2PnW%=;$DgwMZKev1f`*@ zZWl5<21`mu8X3asZ#P{Ym$L@Vo|aM?mOU*&)rAQr21ajr7UW`Ckrz=M{WUZw{*vZ^ zwRf|EA@(3a@RcRB`^(&xuxh&4({KF~e{F74$vTJKY&CUBPy8>ZsRZzo&37TY@`I}( zlRB?NOdhnn8F^530*%4uhoX&P&fp;@p|-Q|{l=1ry?vdo%X*qc-J|{k?r(xL3$qWs z?pYVQ;pR&TzRx#3+FN|yrGcI;+FL^)+_74mW$fyX-q~&LJnrPn+3-2>VD+sZ&{oO% za2;{6E4b=T??%fbx#|}jcFqkE@r`0z4G$6gtk`-kM{#|A{Ui@m@c_z6xeN3WVY?ob zYFN3tj&=&0L5?f00xtG1br*S_r2yGXOb1*O8x-yv{o{7~9pD1m@c;q=*mOZK5!SpVv%=5Q^6leROZNzMA)4Fl64=F*$Ptlc>9$zZzQX6lQJ=UG z756(EDCqNvFOu0{FRb!IJx4%uA4ed0x!TN-fZ($VW(6l5uh7IvSQhUIqp*Nm3E}_x ziU_Irr`G>-z?mFQ>PUOQ85Zmq@}w4Dj*DHb(@y|&l_-k7#EHW~1CyV??1W?`BFOL6 z`+SvTe)#h~#id2#-1jfX`{&n@PVLjzs1C!@Ta!6Tc8I5z9j4-LRWYvqc)~bDUTGQ{ zy=D4Vuw0Cg$r-*Y~98*l0QY|c=TfrQJnxUt#ye=T_->o@W3?rSrgE4AjSSmU|NiK(OFB zKIHv#K-XB)2QMcH)dN6=r43#$z0X;OXLeTAmLUi z569&?=kccrwJ(3O0%&vUTD%*?pT0@Y!&zng0*{q4^JeyZyaL-}s;{lLtj@Ya@FDf2t#h&L z7fe{AU0D@3o-J{kf&DHEu1emHLI-w1>r!beU0w$eY#5$gZ3k4ybP2XtIZj{g9pI)e zRO)ikd8evI%mY8SGj@_M2b!?Q%*1uGM3y_HKN)K5l~>!&GcX#rTdwAXKy3~m5BXeZ zYPN3+xS#d*VNi+YO`K3++kfX%u)XJ?ds%?a$h}D^Y-QRioP_#Mw6r4Hvpef20Z99s z;h8H5yneJ0U_x zq;bt$3c}EP3`_7`uYU`-re%hL4x$G(UlCQ!5u5A3wG6gV7Go};v2+wd+Hdvt1Sm9} zEgB4MvfF>Lk&f?~jm3>}lISS2*etD!^%oA)hK-&L#Rg|%7tE+CJ@4Y9dkSshI0<^k zu$er!T|AWP4a{nx9A0KmSz2!E2_vAd3!u#VXQx=ohg)JN#^m-K{vl;lzVk9ifu7wX zQ8Xtlzg7uqi%{PWUHAMP*{8dUsDsOVoh*7IT*B zWS({Wr&l3qB{bH}k!hTm)Kj#HIG3SFi30D@c)P4pv{cbKoXi1-`(d`}X54HKRASDv zgA{a1*E}+<-5TD}9Gse$wKLlD2F}ZolWXv3QvTO0CzrCniUa7DrO6IYCjM$g2*5zv z{p4zpW5aEm++;Go2WKmxKk%|LN-?vbA%Xeoqz%=OH2ZSS-bQKtVdU=xm`5Y89QDeM z{lp5_*v!sJkt-TlRo%=Mc*gel;o+7zDn0d%dvm$*si5L*$PIeJKUU3|_uJ~tGL}az zK8qbI-}w5PdZX@xU?&>S;N!-bT?+WWNrY-~4E2N4_h=$7v_*DeBDP(m>swIS=E?(GO>e-{m*@fNH;1x5Mtc{>pZu{Q%W|XN^*B?RNsFx6hc57-F0TDZ#Tt<@i6=j2}IH z-Pn%COY?fF&Kif%U`=S|B23udJd0e!5YV5GLODwwY%w(l6p7{lJl&(q!MfR?8YmP! zQP6dolV?qkqvxkqh`(~g@^|n&;p0=||B=(>Aw~d8v}+Y>Hj+DXO)*q_N?SR!3^AK1 z9FGw=9z7i#xdV^`;Q*z%CPq#`hMWVvhX@=mkU4N_9wL$>4>9oZ-2+)U*HhV(?6=sH32=MD*X=zl-FPLaNnm0=swC)B-j{*RaX9Ec70vtBQ+;sxqh~jmV z5M%S~zeE|l8pnqcmn;kgF$6c}W!L1GKXM{sSqvDOgCJk@9N~wPa}efc8z*uQW}9j{ zOnuX#a}cY+xUp(;;@2LjYAqv-kc|IeuD{+<0DO+tNp|LD#`tUVyv3c-J_Fkpsi zbv`+VoKUx&uX{EGYG~~F3N)Gz8VT2+_bgFMn`Oz_?LeTP$4(h%+>zked2Xd~y8YmO zHFSIwUU>R^bMMbBZ^EzRivAo%B3BM0-JioSo50SH!!R3|KqEIlh&j7B)^Ul?*Ua)9 z;R)mG0E~TkoF|;mOAJUe-O~gs=7%YKA*LxV03XMzD8G% zu^!`P08DPMbNSBJ#S9af>+)pT1cIbswvDs0)xL};dca(7fyk1I1(em1mFI&M83)(Qqg$u&09KWeTML3rmhd5fGYMGTR*IMezOQ*-*TXSt}v zp`m#zyOz%{&nIuuZ7$&>Y_1)=whz18a~MplTMSDwYxsaJoY%L;&q{NRpRa++rbcr? zC()y@9i|-2(DO|~&+(R(=S0!-@Z{zdx_Ce2bP{922RHh)amKiFh>nG&$fkq5i zv@C~PlI=O_eV3`lhb%&7KcJJkko(i+ansp9i+bd2bl{!-mU*i+Qvfk1;BouHW{u%c z<;Z=dh$H*^kw;^ct#*)6vO3;)ni^!Y#&B`AkLn6sjK>YYDSZ-7rusZ1PA5e?9_zEG zzQ^&Vfi=#fDw@xXRSXfu?2Bk5;kBmaWb8vIYfITtNnxOpA94Cf#pHi!UC$Hu0SDm9 zL4wMeR2uFbcq9Au!8ZbnfgePWF|BAd8P*Jl@q<@ior$5ofzv3$v{~$QO`6P>LNyK$ zDTu#O<6OIH)f#7w>-H;q?qu4=wziDhvi(MK57>j(!AqD6%r%$NbUh9NMF40HAOZ8% z3Sc%ag8SQ82glmk(F_Kd9WC-mu;cL7{QKhJ!;1^+>kG#(Svqd~ z(xn8}OO3hkZSt*TPK6yH(^;F=O)zQq8Qe6wGgV$#@A5}ijclK>VR6%C>595wSZ;0| zo$mN*I!pI2WL*=>>x9<5vgn!+x08H@0=?9dO_J_1XA1xR01@<~b)h$6L%(gZE=Mo9fIx)*!Vv{a5#6|NqtA^8Y^jZbLJa{1X1XrN-3S zriNt_ayTpo(nRY51?V0ZwCNC4*6vsTIP zawekDf_%T#ogY^15X#(4FOxQA1>(N62RK=}(Ici?wr8(bl|(cW@%ybBx!!@uSI+V> zbx4)NP-bMS#FKj@-fk5cHdIhlta{>ZiDwwb7cC8iMPlO2P}Pbly6lvRQ|J&R4mu#w z`XVP_Y{g8N+@pBuJw#NId%_`_yfg2~=%azOQO!z~j5hv;4m0&YQg0U)7K<3oYxK%P(2Z z#@Uritn;Rj59`yVp^LhZF@JLZ!QB4)Y=;*cwZ{zz!Apw|3uWe~b>${(?F6=m2yC&#Efgx`Gg!Qb?wY0XWN-#qwDivQ2S*H9t)JJHP+ zb%2%e^FvqmeCwf}E58X}zVe&j>gjptklF9^8LkFn%{6isoZl)rL!UX_p0MCcr#pG` zdF~i{nxf*<_lxHD{*$>J{n_HT46}3t2enF(xtC!8)zd5}Eij0ibrdH~5X))cnIZ4! z-@+bbRs*2wnwpoIPW;#;p;p92{Mp(PkTqB6E)>m~P2u})LD@Y7%yr?2F+=HXW>_*6 zapY`0&};fP(XJuC`!~{-K&tN^U~ZzFI`I`Jte?K8#o-@4rTM&Q^B|Y5b8IMtdBMsT zxfo-Lr_|4jqK;f{505}7#BjX87ip+;h&z6PnP_=>cq|r-%ATmm+ld5AsA18jb6{nq zP!S60I)j`MhueyfoGMek0%6^-SUeDuJP}c_Gl*j(QB}%^6?#2PRfk_v-3}2G(HX3Q zdZHFBk`=`zIjo#3j06$HJfcS)H-3XR=<_m_9e!D{0T4}{ICM)QvI3r0WThW4H@E>Q zx+*A&(+PPVbGk!yL#JP`O5lES#w94kaDrwFi$_x2z!|Vw9mdR|Zu*2ld7{KxZM?le zl|m{frlM8#%?=JABCMh=YtZhpA>?YR^S%lsYI51bE=S0(adyq8c{Ekc*BGQIB6ZsB zUYDd4cI9K#oDYf_8XpE67gB9@pKIx|Z}5m*Xp;njr6>DdQ@svvVNlN3{T_Q3kj%Q* z|I79h{@=Fu?w~zcXl#}G3sk~Xz<3F9XXA;;3Hd_*ew)HUWG_)v`<5*UpoWUjB)eUfY^o?Iyu`quYIWh3&6|(E zT-GpdqG~0ipj)qiGLJnj`-mX$bwnc};05*-Q3iKLQ!GN%783YE-60ob*165@ZAl{~ zsbqHZ%5?5sE?xRbi~c3t{d*at2PAB6wPp6W{PHPVVB*qY1Cla#j&;Y)Juz!T)pgl6 zxHZ$~^lg2aJY|Q>x>}a+Prfir*NE(XnTswF*!~Tn|KC{0(vsazkGG5(7;mNPrKeA< zq@E8>p?H53c2JHiTRi?AIx6*Vyl1lwpmkS6V3b)#=(gH)!8&g2Siidi|h^>t(eil1NN9rg4G#X?*Bc^_7b$d{>*%4cL$6a+ILfhAKEx)Ni!kmk1 zONt9-m-CJF^Ytt0zrS${o_!%5SJ;l}dhB#{J~G1xtjA94zRZoK=$f@Oh?*qRiy#sN zHFQkP{TJwRPp{qI?O>x)}ckNm$I zu7TSu(X-3ppD`bQy1e?oFCD(X@B04$bNDG)004NLV_;-pU|^J-#d~^JYCONqR|akl z1`xRO!TJY`{$KIWfrEkF1jyxJU;>E(0DtriUjTTVV_;-pU^n^az~I2a@W0~UX$}Sk zpa=?h0|0){1^9TJZIeG}R6!8N=k4yhh=0Js>L73+q_>M;kxEkt!l7Uzq_MijSfw$f zxFV%sbGD{27U8OBVR2kx8AuU1k%+|Rst5@rcS-I|ezSY0FUAKSvpYMpJKs0+Dla8| z*_nuMVqfdFud$xeV}D(4hleul`&wjO{pFrY?w-gsw=d#86Whh^VAsX@x}Nx^s+&>u zn>u2B2nzh{#e>YwdE+X1F6Qp*Al`-bQzH2icl3F?Pkw_t`x1CR-W`^;2yJ3#^lMm_ zEVT8E^DO@$p7qFT4psbB3HTi~oDzpmkMmpFtIU~|z#D_G$9YqSvF5rCd4J?Vw8p)r z1^gUq?_Y>>@AcHT;L?HXC;j1DaOuR@EUXgOA%2CtdHnO_)W{i7U(0+t3^(;>E@r89 zA#yZMd+vrFM9aITyW19LHBai~e9L(FI-IDhLa%&VHliKx-8;Ewb^^Q0I0unuXk0UW zCuih7FUOkZN$<<3%wZeohyG=t|3|O}-hw6Tk?~KuU4-$4-&U-rgxG^>oQL24%QNJX zX3X?21O2~%O|Svp#2!h~{}Px@de^falo5MCugqFM^EGOHm9&fci4@>_kgNqBfyJ@^ zTX-z9SL}b`e@OVJ_9J-!?i1T%UR}N`(`n9`@icEIX)ecFH<;`7zdiZW$DND&a0lEK zIR|E`AWHxM044w?044w?0Du7w0pbFh1HJ@u1lR=%1wsYn21*8y2FM2Z2Ve)#2oeZ- z2>uCd3Kj~I3g`<~3*HPk45$qt4T=r|4#*FB555ov5IPWW5V{ch5j+u)5x^1p5=0Vq z637z{6Fd`i6Ur1G6l4^d6wDO-6;Ktn75ElJ7WNlh7!DXJ7;YGr7}yyg8B!UD8PFOI z8cZ6X8qgaG8&n&F8`K;a97Y^|9Ml~q9cmq*9nu~E9#9^%9`+wXAF3bzAZ8$nAmkw~ zA%G$hBB&zdBS<5nBl08|BswHuB-|weB`76MC3Gd4CB!BKCLktICV(cUCd?-8Cln_- zCtxThD2yobDNHG@Dn=^aD^M${EEX(+EaEMIEyOMeEAb}HG5k*832`G>bp%nBMSs6=961+|VJOD32MacsoN*;ikl9KN39r$cU z{1I^Ir1NUmznPu+W<6^F3#tl*=Vt?F!U{g_ggs2-QFwy!OTrcSYFl^`i|Sl>ivDZi z8C2DSu#c+uLwFW7@0ai#`GU%-;?p`Qs}!s-j~ihRRon|tU=@FaE0|Z2@FbdQPk4&{ zOW_&(RJX!D`M2;aBCjSqM^3PXV+>*7kRwBi7#dfc%h;tihUKcUNg8pt%eX#zSV2H< zga$&6HKAtdZ8OL6-C=x!L+ZNHDry+k7;)_aTgOA=@+^(@m0sR8u{C-#9;IE^@2vzn zYJ}k$mu8^b*6L209Oh1Uj5FED6xkn+{VIsESi*fve@4?JLmOwGWwo$h9?H=wK1}m7 z0?x)8z1F_TT-xvHFbEspmHtXjW2ZdOY1y}R-lu-)8r~MrJ41V_qxIFI2hH!w{q2(Q zzN`NjTwGj^0XtOZ7?&fUo$_}z6mb+tI32wQlsIBao)22}`glAJ2I+~hE>2<>9Q2R= zv)U6xFsm2<004NLZO~V2l?50E;OB2o2QA1JMG)CTl+#lgmEFqTd&wwLpo~&lDxk7K zR7N07O(3j<74{54NZ5M>REDfOHe4_+u_e9qP2M~g-_`eu@;>isTJHbyG{1o;(L!Rx ziW9Gh1c{0&<^#o*P*N$Sl~LA*%K6C0%KJnG6@99b&s0`LRn=5iLrt~RR>$Y+`ofp$ z2}=@@>?`#(&`={OzSdY1O*PY83oWHe^Nm(oYoo1p+UuaBPCDzNt8TjMp{HJY>!YuJ z`WxU|0}V3R5b4&NZH_N|@Jb6$os&bjP@i!OO&xNEMsYKIYCTI_Gv{bi&lp83@%nMNCHj4W%6GhVhF zx$;c#y@?*1yQM*agFvwo)l004*xBx?Ww literal 0 HcmV?d00001 diff --git a/models/tacotron2/ControllableTalkNet/assets/dash_customfont.css b/models/tacotron2/ControllableTalkNet/assets/dash_customfont.css new file mode 100644 index 0000000000..c9855c1568 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/assets/dash_customfont.css @@ -0,0 +1,420 @@ +@font-face { + font-family: 'EquestriaWebfont'; + src: url('Equestria.woff') format('woff'); +} + +/* Table of contents +–––––––––––––––––––––––––––––––––––––––––––––––––– +- Plotly.js +- Grid +- Base Styles +- Typography +- Links +- Buttons +- Forms +- Lists +- Code +- Tables +- Spacing +- Utilities +- Clearing +- Media Queries +*/ + +/* PLotly.js +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +/* plotly.js's modebar's z-index is 1001 by default + * https://github.com/plotly/plotly.js/blob/7e4d8ab164258f6bd48be56589dacd9bdd7fded2/src/css/_modebar.scss#L5 + * In case a dropdown is above the graph, the dropdown's options + * will be rendered below the modebar + * Increase the select option's z-index + */ + +/* This was actually not quite right - + dropdowns were overlapping each other (edited October 26) + +.Select { + z-index: 1002; +}*/ + + +/* Grid +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +.container { + position: relative; + width: 100%; + max-width: 960px; + margin: 0 auto; + padding: 0 20px; + box-sizing: border-box; } +.column, +.columns { + width: 100%; + float: left; + box-sizing: border-box; } + +/* For devices larger than 400px */ +@media (min-width: 400px) { + .container { + width: 85%; + padding: 0; } +} + +/* For devices larger than 550px */ +@media (min-width: 550px) { + .container { + width: 80%; } + .column, + .columns { + margin-left: 4%; } + .column:first-child, + .columns:first-child { + margin-left: 0; } + + .one.column, + .one.columns { width: 4.66666666667%; } + .two.columns { width: 13.3333333333%; } + .three.columns { width: 22%; } + .four.columns { width: 30.6666666667%; } + .five.columns { width: 39.3333333333%; } + .six.columns { width: 48%; } + .seven.columns { width: 56.6666666667%; } + .eight.columns { width: 65.3333333333%; } + .nine.columns { width: 74.0%; } + .ten.columns { width: 82.6666666667%; } + .eleven.columns { width: 91.3333333333%; } + .twelve.columns { width: 100%; margin-left: 0; } + + .one-third.column { width: 30.6666666667%; } + .two-thirds.column { width: 65.3333333333%; } + + .one-half.column { width: 48%; } + + /* Offsets */ + .offset-by-one.column, + .offset-by-one.columns { margin-left: 8.66666666667%; } + .offset-by-two.column, + .offset-by-two.columns { margin-left: 17.3333333333%; } + .offset-by-three.column, + .offset-by-three.columns { margin-left: 26%; } + .offset-by-four.column, + .offset-by-four.columns { margin-left: 34.6666666667%; } + .offset-by-five.column, + .offset-by-five.columns { margin-left: 43.3333333333%; } + .offset-by-six.column, + .offset-by-six.columns { margin-left: 52%; } + .offset-by-seven.column, + .offset-by-seven.columns { margin-left: 60.6666666667%; } + .offset-by-eight.column, + .offset-by-eight.columns { margin-left: 69.3333333333%; } + .offset-by-nine.column, + .offset-by-nine.columns { margin-left: 78.0%; } + .offset-by-ten.column, + .offset-by-ten.columns { margin-left: 86.6666666667%; } + .offset-by-eleven.column, + .offset-by-eleven.columns { margin-left: 95.3333333333%; } + + .offset-by-one-third.column, + .offset-by-one-third.columns { margin-left: 34.6666666667%; } + .offset-by-two-thirds.column, + .offset-by-two-thirds.columns { margin-left: 69.3333333333%; } + + .offset-by-one-half.column, + .offset-by-one-half.columns { margin-left: 52%; } + +} + + +/* Base Styles +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +/* NOTE +html is set to 62.5% so that all the REM measurements throughout Skeleton +are based on 10px sizing. So basically 1.5rem = 15px :) */ +html { + font-size: 62.5%; } +body { + font-size: 1.5em; /* currently ems cause chrome bug misinterpreting rems on body element */ + line-height: 1.6; + font-weight: 400; + font-family: "Open Sans", "HelveticaNeue", "Helvetica Neue", Helvetica, Arial, sans-serif; + color: rgb(50, 50, 50); } + + +/* Typography +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +h1, h2, h3, h4, h5, h6 { + margin-top: 0; + margin-bottom: 0; + font-weight: 300; } +h1 { font-size: 4.5rem; line-height: 1.2; letter-spacing: -.1rem; margin-bottom: 2rem; } +h2 { font-size: 3.6rem; line-height: 1.25; letter-spacing: -.1rem; margin-bottom: 1.8rem; margin-top: 1.8rem;} +h3 { font-size: 3.0rem; line-height: 1.3; letter-spacing: -.1rem; margin-bottom: 1.5rem; margin-top: 1.5rem;} +h4 { font-size: 2.6rem; line-height: 1.35; letter-spacing: -.08rem; margin-bottom: 1.2rem; margin-top: 1.2rem;} +h5 { font-size: 2.2rem; line-height: 1.5; letter-spacing: -.05rem; margin-bottom: 0.6rem; margin-top: 0.6rem;} +h6 { font-size: 2.0rem; line-height: 1.6; letter-spacing: 0; margin-bottom: 0.75rem; margin-top: 0.75rem;} + +p { + margin-top: 0; } + + +/* Blockquotes +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +blockquote { + border-left: 4px lightgrey solid; + padding-left: 1rem; + margin-top: 2rem; + margin-bottom: 2rem; + margin-left: 0rem; +} + + +/* Links +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +a { + color: #1EAEDB; + text-decoration: underline; + cursor: pointer;} +a:hover { + color: #0FA0CE; } + + +/* Buttons +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +.button, +button, +input[type="submit"], +input[type="reset"], +input[type="button"] { + display: inline-block; + height: 38px; + padding: 0 30px; + color: #555; + text-align: center; + font-size: 11px; + font-weight: 600; + line-height: 38px; + letter-spacing: .1rem; + text-transform: uppercase; + text-decoration: none; + white-space: nowrap; + background-color: transparent; + border-radius: 4px; + border: 1px solid #bbb; + cursor: pointer; + box-sizing: border-box; } +.button:hover, +button:hover, +input[type="submit"]:hover, +input[type="reset"]:hover, +input[type="button"]:hover, +.button:focus, +button:focus, +input[type="submit"]:focus, +input[type="reset"]:focus, +input[type="button"]:focus { + color: #333; + border-color: #888; + outline: 0; } +.button.button-primary, +button.button-primary, +input[type="submit"].button-primary, +input[type="reset"].button-primary, +input[type="button"].button-primary { + color: #FFF; + background-color: #33C3F0; + border-color: #33C3F0; } +.button.button-primary:hover, +button.button-primary:hover, +input[type="submit"].button-primary:hover, +input[type="reset"].button-primary:hover, +input[type="button"].button-primary:hover, +.button.button-primary:focus, +button.button-primary:focus, +input[type="submit"].button-primary:focus, +input[type="reset"].button-primary:focus, +input[type="button"].button-primary:focus { + color: #FFF; + background-color: #1EAEDB; + border-color: #1EAEDB; } + + +/* Forms +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +input[type="email"], +input[type="number"], +input[type="search"], +input[type="text"], +input[type="tel"], +input[type="url"], +input[type="password"], +textarea, +select { + height: 38px; + padding: 6px 10px; /* The 6px vertically centers text on FF, ignored by Webkit */ + background-color: #fff; + border: 1px solid #D1D1D1; + border-radius: 4px; + box-shadow: none; + box-sizing: border-box; + font-family: inherit; + font-size: inherit; /*https://stackoverflow.com/questions/6080413/why-doesnt-input-inherit-the-font-from-body*/} +/* Removes awkward default styles on some inputs for iOS */ +input[type="email"], +input[type="number"], +input[type="search"], +input[type="text"], +input[type="tel"], +input[type="url"], +input[type="password"], +textarea { + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; } +textarea { + min-height: 65px; + padding-top: 6px; + padding-bottom: 6px; } +input[type="email"]:focus, +input[type="number"]:focus, +input[type="search"]:focus, +input[type="text"]:focus, +input[type="tel"]:focus, +input[type="url"]:focus, +input[type="password"]:focus, +textarea:focus, +select:focus { + border: 1px solid #33C3F0; + outline: 0; } +label, +legend { + display: block; + margin-bottom: 0px; } +fieldset { + padding: 0; + border-width: 0; } +input[type="checkbox"], +input[type="radio"] { + display: inline; } +label > .label-body { + display: inline-block; + margin-left: .5rem; + font-weight: normal; } + + +/* Lists +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +ul { + list-style: circle inside; } +ol { + list-style: decimal inside; } +ol, ul { + padding-left: 0; + margin-top: 0; } +ul ul, +ul ol, +ol ol, +ol ul { + margin: 1.5rem 0 1.5rem 3rem; + font-size: 90%; } +li { + margin-bottom: 1rem; } + + +/* Tables +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +table { + border-collapse: collapse; +} +th:not(.CalendarDay), +td:not(.CalendarDay) { + padding: 12px 15px; + text-align: left; + border-bottom: 1px solid #E1E1E1; } +th:first-child:not(.CalendarDay), +td:first-child:not(.CalendarDay) { + padding-left: 0; } +th:last-child:not(.CalendarDay), +td:last-child:not(.CalendarDay) { + padding-right: 0; } + + +/* Spacing +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +button, +.button { + margin-bottom: 0rem; } +input, +textarea, +select, +fieldset { + margin-bottom: 0rem; } +pre, +dl, +figure, +table, +form { + margin-bottom: 0rem; } +p, +ul, +ol { + margin-bottom: 0.75rem; } + +/* Utilities +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +.u-full-width { + width: 100%; + box-sizing: border-box; } +.u-max-full-width { + max-width: 100%; + box-sizing: border-box; } +.u-pull-right { + float: right; } +.u-pull-left { + float: left; } + + +/* Misc +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +hr { + margin-top: 3rem; + margin-bottom: 3.5rem; + border-width: 0; + border-top: 1px solid #E1E1E1; } + + +/* Clearing +–––––––––––––––––––––––––––––––––––––––––––––––––– */ + +/* Self Clearing Goodness */ +.container:after, +.row:after, +.u-cf { + content: ""; + display: table; + clear: both; } + + +/* Media Queries +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +/* +Note: The best way to structure the use of media queries is to create the queries +near the relevant code. For example, if you wanted to change the styles for buttons +on small devices, paste the mobile query code up in the buttons section and style it +there. +*/ + + +/* Larger than mobile */ +@media (min-width: 400px) {} + +/* Larger than phablet (also point when grid becomes active) */ +@media (min-width: 550px) {} + +/* Larger than tablet */ +@media (min-width: 750px) {} + +/* Larger than desktop */ +@media (min-width: 1000px) {} + +/* Larger than Desktop HD */ +@media (min-width: 1200px) {} \ No newline at end of file diff --git a/models/tacotron2/ControllableTalkNet/controllable_talknet.py b/models/tacotron2/ControllableTalkNet/controllable_talknet.py new file mode 100644 index 0000000000..3f68cc0077 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/controllable_talknet.py @@ -0,0 +1,1097 @@ +import sys +import os +import base64 +import dash +from jupyter_dash import JupyterDash +import dash_core_components as dcc +import dash_html_components as html +import dash_bootstrap_components as dbc +from dash.exceptions import PreventUpdate +import torch +import numpy as np +import tensorflow as tf +import crepe +import scipy +from scipy.io import wavfile +import psola +import io +import nemo +from nemo.collections.asr.models import EncDecCTCModel +from nemo.collections.tts.models import TalkNetSpectModel +from nemo.collections.tts.models import TalkNetPitchModel +from nemo.collections.tts.models import TalkNetDursModel +from talknet_singer import TalkNetSingerModel +import json +from tqdm import tqdm +import gdown +import zipfile +import resampy +import traceback +import ffmpeg +import time +import uuid + +sys.path.append("hifi-gan") +from env import AttrDict +from meldataset import mel_spectrogram, MAX_WAV_VALUE +from models import Generator +from denoiser import Denoiser + +app = JupyterDash(__name__) +DEVICE = "cuda:0" +CPU_PITCH = False +RUN_PATH = os.path.dirname(os.path.realpath(__file__)) +if RUN_PATH == "/content": + UI_MODE = "colab" +elif os.path.exists("/talknet/is_docker"): + UI_MODE = "docker" +else: + UI_MODE = "offline" +torch.set_grad_enabled(False) +if CPU_PITCH: + tf.config.set_visible_devices([], "GPU") + +app.title = "Controllable TalkNet" +app.layout = html.Div( + children=[ + html.H1( + id="header", + children="Controllable TalkNet", + style={ + "font-family": "EquestriaWebfont", + "color": "#ed3c96", + "font-size": "4em", + "text-align": "center", + "margin-top": "0em", + "margin-bottom": "0em", + }, + ), + html.Label("Character selection", htmlFor="model-dropdown"), + dbc.Select( + id="model-dropdown", + options=[ + { + "label": "Custom model", + "value": "Custom", + }, + { + "label": "--- ERROR LOADING MODEL LISTS ---", + "value": "", + "disabled": True, + }, + ], + value=None, + style={ + "max-width": "90vw", + "width": "35em", + "margin-bottom": "0.7em", + }, + ), + html.Div( + children=[ + dcc.Input( + id="drive-id", + type="text", + placeholder="Drive ID for custom model", + style={"width": "22em"}, + ), + ], + id="custom-model", + style={ + "display": "none", + }, + ), + html.Label( + "Upload reference audio to " + RUN_PATH, + htmlFor="reference-dropdown", + id="upload-label", + ), + dcc.Upload( + id="upload-audio", + children=html.Div(["Drag and drop or click to select a file to upload."]), + style={ + "display": "none", + }, + multiple=True, + ), + dcc.Store(id="current-f0s"), + dcc.Store(id="current-f0s-nosilence"), + dcc.Store(id="current-filename"), + dcc.Loading( + id="audio-loading", + children=[ + html.Div( + [ + html.Button( + "Update file list", + id="update-button", + style={ + "margin-right": "10px", + }, + ), + dbc.Select( + id="reference-dropdown", + options=[], + value=None, + style={ + "max-width": "80vw", + "width": "30em", + }, + disabled=False, + ), + dcc.Store(id="pitch-clicks"), + html.Button( + "Debug pitch", + id="pitch-button", + style={ + "margin-left": "10px", + }, + disabled=False, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "row", + "margin-left": "50px", + "vertical-align": "middle", + }, + ), + html.Audio( + id="pitch-out", + controls=True, + style={"display": "none"}, + ), + html.Div( + id="audio-loading-output", + style={ + "font-style": "italic", + "margin-bottom": "0.7em", + "text-align": "center", + }, + ), + ], + type="default", + ), + html.Div( + [ + dcc.Checklist( + id="pitch-options", + options=[ + {"label": "Change input pitch", "value": "pf"}, + {"label": "Auto-tune output", "value": "pc"}, + {"label": "Disable reference audio", "value": "dra"}, + ], + value=[], + ), + html.Div( + [ + html.Label("Semitones", htmlFor="pitch-factor"), + dcc.Input( + id="pitch-factor", + type="number", + value="0", + style={"width": "7em"}, + min=-11, + max=11, + step=1, + disabled=True, + ), + ], + style={ + "flex-direction": "column", + "margin-left": "10px", + "margin-bottom": "0.7em", + }, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "row", + "margin-left": "50px", + "margin-bottom": "0.7em", + }, + ), + html.Label("Transcript", htmlFor="transcript-input"), + dcc.Textarea( + id="transcript-input", + value="", + style={ + "max-width": "90vw", + "width": "50em", + "height": "8em", + "margin-bottom": "0.7em", + }, + ), + dcc.Loading( + html.Div( + [ + html.Button( + "Generate", + id="gen-button", + ), + html.Audio( + id="audio-out", + controls=True, + style={ + "display": "none", + }, + ), + html.Div( + id="generated-info", + style={ + "font-style": "italic", + }, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "column", + }, + ) + ), + html.Footer( + children=""" + Presented by the Pony Preservation Project. + """, + style={"margin-top": "2em", "font-size": "0.7em"}, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "column", + "background-color": "#FFF", + }, +) + +upload_display = { + "width": "100%", + "height": "60px", + "lineHeight": "60px", + "borderWidth": "1px", + "borderStyle": "dashed", + "borderRadius": "5px", + "textAlign": "center", + "margin": "10px", +} + +playback_style = { + "margin-top": "0.3em", + "margin-bottom": "0.3em", + "display": "block", + "width": "600px", + "max-width": "90vw", +} + +playback_hide = { + "display": "none", +} + + +@app.callback( + [ + dash.dependencies.Output("model-dropdown", "options"), + dash.dependencies.Output("upload-audio", "style"), + ], + dash.dependencies.Input("header", "children"), +) +def init_dropdown(value): + if UI_MODE == "docker": + upload_style = upload_display + else: + upload_style = playback_hide + + dropdown = [ + { + "label": "Custom model", + "value": "Custom|default", + } + ] + prev_values = ["Custom|default"] + + def add_to_dropdown(entry): + if entry["value"] in prev_values: + return + dropdown.append(entry) + prev_values.append(entry["value"]) + + all_dict = {} + for filename in os.listdir("model_lists"): + if len(filename) < 5 or filename[-5:].lower() != ".json": + continue + with open(os.path.join("model_lists", filename)) as f: + j = json.load(f) + for s in j: + for c in s["characters"]: + c["source_file"] = filename[:-5] + if s["source"] not in all_dict: + all_dict[s["source"]] = s["characters"] + else: + all_dict[s["source"]].extend(s["characters"]) + for k in sorted(all_dict): + seen_chars = [] + seen_ids = [] + characters = {} + characters_sing = {} + has_singers = False + for c in all_dict[k]: + if c["drive_id"] in seen_ids: + continue + seen_ids.append(c["drive_id"]) + # Handle duplicate names + if c["name"] in seen_chars: + if c["name"] in characters: + rename = ( + c["name"] + " [" + characters[c["name"]]["source_file"] + "]" + ) + characters[rename] = characters[c["name"]] + del characters[c["name"]] + c["name"] = c["name"] + " [" + c["source_file"] + "]" + else: + seen_chars.append(c["name"]) + + characters[c["name"]] = { + "drive_id": c["drive_id"], + "is_singing": c["is_singing"], + "source_file": c["source_file"], + } + if c["is_singing"]: + has_singers = True + if has_singers: + for ck in sorted(characters): + if characters[ck]["is_singing"]: + characters_sing[ck] = characters[ck] + del characters[ck] + separator = "--- " + k.strip().upper() + " MODELS (TALKING) ---" + else: + separator = "--- " + k.strip().upper() + " MODELS ---" + if len(characters) > 0: + add_to_dropdown( + { + "label": separator, + "value": str(uuid.uuid4()) + "|default", + "disabled": True, + } + ) + for ck in sorted(characters): + add_to_dropdown( + { + "label": ck, + "value": characters[ck]["drive_id"] + "|default", + } + ) + if has_singers: + separator = "--- " + k.strip().upper() + " MODELS (SINGING) ---" + add_to_dropdown( + { + "label": separator, + "value": str(uuid.uuid4()) + "|default", + "disabled": True, + } + ) + for ck in sorted(characters_sing): + add_to_dropdown( + { + "label": ck, + "value": characters_sing[ck]["drive_id"] + "|singing", + } + ) + if len(all_dict) == 0: + add_to_dropdown( + { + "label": "--- NO MODEL LISTS FOUND ---", + "value": str(uuid.uuid4()) + "|default", + "disabled": True, + } + ) + return [dropdown, upload_style] + + +def load_hifigan(model_name, conf_name): + # Load HiFi-GAN + conf = os.path.join("hifi-gan", conf_name + ".json") + with open(conf) as f: + json_config = json.loads(f.read()) + h = AttrDict(json_config) + torch.manual_seed(h.seed) + hifigan = Generator(h).to(torch.device(DEVICE)) + state_dict_g = torch.load(model_name, map_location=torch.device(DEVICE)) + hifigan.load_state_dict(state_dict_g["generator"]) + hifigan.eval() + hifigan.remove_weight_norm() + denoiser = Denoiser(hifigan, mode="normal") + return hifigan, h, denoiser + + +def generate_json(input, outpath): + output = "" + sample_rate = 22050 + lpath = input.split("|")[0].strip() + size = os.stat(lpath).st_size + x = { + "audio_filepath": lpath, + "duration": size / (sample_rate * 2), + "text": input.split("|")[1].strip(), + } + output += json.dumps(x) + "\n" + with open(outpath, "w", encoding="utf8") as w: + w.write(output) + + +asr_model = ( + EncDecCTCModel.from_pretrained(model_name="asr_talknet_aligner").cpu().eval() +) + + +def forward_extractor(tokens, log_probs, blank): + """Computes states f and p.""" + n, m = len(tokens), log_probs.shape[0] + # `f[s, t]` -- max sum of log probs for `s` first codes + # with `t` first timesteps with ending in `tokens[s]`. + f = np.empty((n + 1, m + 1), dtype=float) + f.fill(-(10 ** 9)) + p = np.empty((n + 1, m + 1), dtype=int) + f[0, 0] = 0.0 # Start + for s in range(1, n + 1): + c = tokens[s - 1] + for t in range((s + 1) // 2, m + 1): + f[s, t] = log_probs[t - 1, c] + # Option #1: prev char is equal to current one. + if s == 1 or c == blank or c == tokens[s - 3]: + options = f[s : (s - 2 if s > 1 else None) : -1, t - 1] + else: # Is not equal to current one. + options = f[s : (s - 3 if s > 2 else None) : -1, t - 1] + f[s, t] += np.max(options) + p[s, t] = np.argmax(options) + return f, p + + +def backward_extractor(f, p): + """Computes durs from f and p.""" + n, m = f.shape + n -= 1 + m -= 1 + durs = np.zeros(n, dtype=int) + if f[-1, -1] >= f[-2, -1]: + s, t = n, m + else: + s, t = n - 1, m + while s > 0: + durs[s - 1] += 1 + s -= p[s, t] + t -= 1 + assert durs.shape[0] == n + assert np.sum(durs) == m + assert np.all(durs[1::2] > 0) + return durs + + +def preprocess_tokens(tokens, blank): + new_tokens = [blank] + for c in tokens: + new_tokens.extend([c, blank]) + tokens = new_tokens + return tokens + + +parser = ( + nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset.make_vocab( + notation="phonemes", + punct=True, + spaces=True, + stresses=False, + add_blank_at="last", + ) +) + + +def arpa_parse(input, model): + z = [] + space = parser.labels.index(" ") + while "{" in input: + if "}" not in input: + input.replace("{", "") + else: + pre = input[: input.find("{")] + if pre.strip() != "": + x = model.parse(text=pre.strip()) + seq_ids = x.squeeze(0).cpu().detach().numpy() + z.extend(seq_ids) + z.append(space) + + arpaword = input[input.find("{") + 1 : input.find("}")] + arpaword = ( + arpaword.replace("0", "") + .replace("1", "") + .replace("2", "") + .strip() + .split(" ") + ) + + seq_ids = [] + for x in arpaword: + if x == "": + continue + if x.replace("_", " ") not in parser.labels: + continue + seq_ids.append(parser.labels.index(x.replace("_", " "))) + seq_ids.append(space) + z.extend(seq_ids) + input = input[input.find("}") + 1 :] + if input != "": + x = model.parse(text=input.strip()) + seq_ids = x.squeeze(0).cpu().detach().numpy() + z.extend(seq_ids) + if z[-1] == space: + z = z[:-1] + if z[0] == space: + z = z[1:] + return [ + z[i] for i in range(len(z)) if (i == 0) or (z[i] != z[i - 1]) or (z[i] != space) + ] + + +def to_arpa(input): + arpa = "" + z = [] + space = parser.labels.index(" ") + while space in input: + z.append(input[: input.index(space)]) + input = input[input.index(space) + 1 :] + z.append(input) + for y in z: + if len(y) == 0: + continue + + arpaword = " {" + for s in y: + if parser.labels[s] == " ": + arpaword += "_ " + else: + arpaword += parser.labels[s] + " " + arpaword += "} " + if not arpaword.replace("{", "").replace("}", "").replace(" ", "").isalnum(): + arpaword = arpaword.replace("{", "").replace(" }", "") + arpa += arpaword + return arpa.replace(" ", " ").replace(" }", "}").strip() + + +def get_duration(wav_name, transcript, tokens): + if not os.path.exists(os.path.join(RUN_PATH, "temp")): + os.mkdir(os.path.join(RUN_PATH, "temp")) + if "_" not in transcript: + generate_json( + os.path.join(RUN_PATH, "temp", wav_name + "_conv.wav") + + "|" + + transcript.strip(), + os.path.join(RUN_PATH, "temp", wav_name + ".json"), + ) + else: + generate_json( + os.path.join(RUN_PATH, "temp", wav_name + "_conv.wav") + "|" + "dummy", + os.path.join(RUN_PATH, "temp", wav_name + ".json"), + ) + + data_config = { + "manifest_filepath": os.path.join(RUN_PATH, "temp", wav_name + ".json"), + "sample_rate": 22050, + "batch_size": 1, + } + + dataset = nemo.collections.asr.data.audio_to_text._AudioTextDataset( + manifest_filepath=data_config["manifest_filepath"], + sample_rate=data_config["sample_rate"], + parser=parser, + ) + + dl = torch.utils.data.DataLoader( + dataset=dataset, + batch_size=data_config["batch_size"], + collate_fn=dataset.collate_fn, + shuffle=False, + ) + + blank_id = asr_model.decoder.num_classes_with_blank - 1 + + for sample_idx, test_sample in tqdm(enumerate(dl), total=len(dl)): + log_probs, _, greedy_predictions = asr_model( + input_signal=test_sample[0], input_signal_length=test_sample[1] + ) + + log_probs = log_probs[0].cpu().detach().numpy() + target_tokens = preprocess_tokens(tokens, blank_id) + + f, p = forward_extractor(target_tokens, log_probs, blank_id) + durs = backward_extractor(f, p) + + del test_sample + return durs + return None + + +def crepe_f0(wav_path, hop_length=256): + # sr, audio = wavfile.read(io.BytesIO(wav_data)) + sr, audio = wavfile.read(wav_path) + audio_x = np.arange(0, len(audio)) / 22050.0 + f0time, frequency, confidence, activation = crepe.predict(audio, sr, viterbi=True) + + x = np.arange(0, len(audio), hop_length) / 22050.0 + freq_interp = np.interp(x, f0time, frequency) + conf_interp = np.interp(x, f0time, confidence) + audio_interp = np.interp(x, audio_x, np.absolute(audio)) / 32768.0 + weights = [0.5, 0.25, 0.25] + audio_smooth = np.convolve(audio_interp, np.array(weights)[::-1], "same") + + conf_threshold = 0.25 + audio_threshold = 0.0005 + for i in range(len(freq_interp)): + if conf_interp[i] < conf_threshold: + freq_interp[i] = 0.0 + if audio_smooth[i] < audio_threshold: + freq_interp[i] = 0.0 + + # Hack to make f0 and mel lengths equal + if len(audio) % hop_length == 0: + freq_interp = np.pad(freq_interp, pad_width=[0, 1]) + return ( + torch.from_numpy(freq_interp.astype(np.float32)), + torch.from_numpy(frequency.astype(np.float32)), + ) + + +def f0_to_audio(f0s): + volume = 0.2 + sr = 22050 + freq = 440.0 + base_audio = ( + np.sin(2 * np.pi * np.arange(256.0 * len(f0s)) * freq / sr) * volume + ).astype(np.float32) + shifted_audio = psola.vocode(base_audio, sr, target_pitch=f0s) + for i in range(len(f0s)): + if f0s[i] == 0.0: + shifted_audio[i * 256 : (i + 1) * 256] = 0.0 + print(type(shifted_audio[0])) + buffer = io.BytesIO() + wavfile.write(buffer, sr, shifted_audio.astype(np.float32)) + b64 = base64.b64encode(buffer.getvalue()) + sound = "data:audio/x-wav;base64," + b64.decode("ascii") + return sound + + +@app.callback( + dash.dependencies.Output("custom-model", "style"), + dash.dependencies.Input("model-dropdown", "value"), +) +def update_model(model): + if model is not None and model.split("|")[0] == "Custom": + style = {"margin-bottom": "0.7em", "display": "block"} + else: + style = {"display": "none"} + return style + + +@app.callback( + [ + dash.dependencies.Output("pitch-factor", "disabled"), + dash.dependencies.Output("reference-dropdown", "disabled"), + dash.dependencies.Output("pitch-button", "disabled"), + ], + [ + dash.dependencies.Input("pitch-options", "value"), + ], +) +def update_pitch_options(value): + return ["pf" not in value, "dra" in value, "dra" in value] + + +@app.callback( + dash.dependencies.Output("upload-label", "children"), + [ + dash.dependencies.Input("upload-audio", "filename"), + dash.dependencies.Input("upload-audio", "contents"), + ], +) +def save_upload(uploaded_filenames, uploaded_file_contents): + try: + if uploaded_filenames is not None and uploaded_file_contents is not None: + for name, content in zip(uploaded_filenames, uploaded_file_contents): + if name.strip() == "": + continue + data = content.encode("utf8").split(b";base64,")[1] + with open(os.path.join(RUN_PATH, name), "wb") as fp: + fp.write(base64.decodebytes(data)) + except Exception as e: + return str(e) + return "Uploaded " + str(len(uploaded_filenames)) + " file(s)" + + +@app.callback( + dash.dependencies.Output("reference-dropdown", "options"), + [ + dash.dependencies.Input("update-button", "n_clicks"), + ], +) +def update_filelist(n_clicks): + filelist = [] + supported_formats = [".wav", ".ogg", ".mp3", "flac", ".aac"] + for x in sorted(os.listdir(RUN_PATH)): + if x[-4:].lower() in supported_formats: + filelist.append({"label": x, "value": x}) + return filelist + + +@app.callback( + [ + dash.dependencies.Output("audio-loading-output", "children"), + dash.dependencies.Output("current-f0s", "data"), + dash.dependencies.Output("current-f0s-nosilence", "data"), + dash.dependencies.Output("current-filename", "data"), + ], + [ + dash.dependencies.Input("reference-dropdown", "value"), + ], +) +def select_file(dropdown_value): + if dropdown_value is not None: + if not os.path.exists(os.path.join(RUN_PATH, "temp")): + os.mkdir(os.path.join(RUN_PATH, "temp")) + ffmpeg.input(os.path.join(RUN_PATH, dropdown_value)).output( + os.path.join(RUN_PATH, "temp", dropdown_value + "_conv.wav"), + ar="22050", + ac="1", + acodec="pcm_s16le", + map_metadata="-1", + fflags="+bitexact", + ).overwrite_output().run(quiet=True) + fo_with_silence, f0_wo_silence = crepe_f0( + os.path.join(RUN_PATH, "temp", dropdown_value + "_conv.wav") + ) + return [ + "Analyzed " + dropdown_value, + fo_with_silence, + f0_wo_silence, + dropdown_value, + ] + else: + return ["No audio analyzed", None, None] + + +@app.callback( + [ + dash.dependencies.Output("pitch-out", "src"), + dash.dependencies.Output("pitch-out", "style"), + dash.dependencies.Output("pitch-clicks", "data"), + ], + [ + dash.dependencies.Input("pitch-button", "n_clicks"), + dash.dependencies.Input("pitch-clicks", "data"), + dash.dependencies.Input("current-f0s", "data"), + ], +) +def debug_pitch(n_clicks, pitch_clicks, current_f0s): + if not n_clicks or current_f0s is None or n_clicks <= pitch_clicks: + if n_clicks is not None: + pitch_clicks = n_clicks + else: + pitch_clicks = 0 + return [ + None, + playback_hide, + pitch_clicks, + ] + pitch_clicks = n_clicks + return [f0_to_audio(current_f0s), playback_style, pitch_clicks] + + +hifigan_sr = None + + +def download_model(model, custom_model): + try: + global hifigan_sr, h2, denoiser_sr + d = "https://drive.google.com/uc?id=" + if model == "Custom": + drive_id = custom_model + else: + drive_id = model + if drive_id == "" or drive_id is None: + return ("Missing Drive ID", None, None) + if not os.path.exists(os.path.join(RUN_PATH, "models")): + os.mkdir(os.path.join(RUN_PATH, "models")) + if not os.path.exists(os.path.join(RUN_PATH, "models", drive_id)): + os.mkdir(os.path.join(RUN_PATH, "models", drive_id)) + zip_path = os.path.join(RUN_PATH, "models", drive_id, "model.zip") + gdown.download( + d + drive_id, + zip_path, + quiet=False, + ) + if not os.path.exists(zip_path): + os.rmdir(os.path.join(RUN_PATH, "models", drive_id)) + return ("Model download failed", None, None) + if os.stat(zip_path).st_size < 16: + os.remove(zip_path) + os.rmdir(os.path.join(RUN_PATH, "models", drive_id)) + return ("Model zip is empty", None, None) + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(os.path.join(RUN_PATH, "models", drive_id)) + os.remove(zip_path) + + # Download super-resolution HiFi-GAN + sr_path = "hifi-gan/hifisr" + if not os.path.exists(sr_path): + gdown.download( + d + "14fOprFAIlCQkVRxsfInhEPG0n-xN4QOa", sr_path, quiet=False + ) + if not os.path.exists(sr_path): + raise Exception("HiFI-GAN model failed to download!") + if hifigan_sr is None: + hifigan_sr, h2, denoiser_sr = load_hifigan(sr_path, "config_32k") + + return ( + None, + os.path.join(RUN_PATH, "models", drive_id, "TalkNetSpect.nemo"), + os.path.join(RUN_PATH, "models", drive_id, "hifiganmodel"), + ) + except Exception as e: + return (str(e), None, None) + + +tnmodel, tnpath, tndurs, tnpitch = None, None, None, None +hifigan, h, denoiser, hifipath = None, None, None, None + + +@app.callback( + [ + dash.dependencies.Output("audio-out", "src"), + dash.dependencies.Output("generated-info", "children"), + dash.dependencies.Output("audio-out", "style"), + dash.dependencies.Output("audio-out", "title"), + ], + [dash.dependencies.Input("gen-button", "n_clicks")], + [ + dash.dependencies.State("model-dropdown", "value"), + dash.dependencies.State("drive-id", "value"), + dash.dependencies.State("transcript-input", "value"), + dash.dependencies.State("pitch-options", "value"), + dash.dependencies.State("pitch-factor", "value"), + dash.dependencies.State("current-filename", "data"), + dash.dependencies.State("current-f0s", "data"), + dash.dependencies.State("current-f0s-nosilence", "data"), + ], +) +def generate_audio( + n_clicks, + model, + custom_model, + transcript, + pitch_options, + pitch_factor, + wav_name, + f0s, + f0s_wo_silence, +): + global tnmodel, tnpath, tndurs, tnpitch, hifigan, h, denoiser, hifipath + + if n_clicks is None: + raise PreventUpdate + if model is None: + return [None, "No character selected", playback_hide, None] + if transcript is None or transcript.strip() == "": + return [ + None, + "No transcript entered", + playback_hide, + None, + ] + if wav_name is None and "dra" not in pitch_options: + return [ + None, + "No reference audio selected", + playback_hide, + None, + ] + load_error, talknet_path, hifigan_path = download_model( + model.split("|")[0], custom_model + ) + if load_error is not None: + return [ + None, + load_error, + playback_hide, + None, + ] + + try: + with torch.no_grad(): + if tnpath != talknet_path: + singer_path = os.path.join( + os.path.dirname(talknet_path), "TalkNetSinger.nemo" + ) + if os.path.exists(singer_path): + tnmodel = TalkNetSingerModel.restore_from(singer_path) + else: + tnmodel = TalkNetSpectModel.restore_from(talknet_path) + durs_path = os.path.join( + os.path.dirname(talknet_path), "TalkNetDurs.nemo" + ) + pitch_path = os.path.join( + os.path.dirname(talknet_path), "TalkNetPitch.nemo" + ) + if os.path.exists(durs_path): + tndurs = TalkNetDursModel.restore_from(durs_path) + tnmodel.add_module("_durs_model", tndurs) + tnpitch = TalkNetPitchModel.restore_from(pitch_path) + tnmodel.add_module("_pitch_model", tnpitch) + else: + tndurs = None + tnpitch = None + tnmodel.eval() + tnpath = talknet_path + + token_list = arpa_parse(transcript, tnmodel) + tokens = torch.IntTensor(token_list).view(1, -1).to(DEVICE) + arpa = to_arpa(token_list) + + if "dra" in pitch_options: + if tndurs is None or tnpitch is None: + return [ + None, + "Model doesn't support pitch prediction", + playback_hide, + None, + ] + spect = tnmodel.generate_spectrogram(tokens=tokens) + else: + durs = get_duration(wav_name, transcript, token_list) + + # Change pitch + if "pf" in pitch_options: + f0_factor = np.power(np.e, (0.0577623 * float(pitch_factor))) + f0s = [x * f0_factor for x in f0s] + f0s_wo_silence = [x * f0_factor for x in f0s_wo_silence] + + spect = tnmodel.force_spectrogram( + tokens=tokens, + durs=torch.from_numpy(durs) + .view(1, -1) + .type(torch.LongTensor) + .to(DEVICE), + f0=torch.FloatTensor(f0s).view(1, -1).to(DEVICE), + ) + + if hifipath != hifigan_path: + hifigan, h, denoiser = load_hifigan(hifigan_path, "config_v1") + hifipath = hifigan_path + + y_g_hat = hifigan(spect.float()) + audio = y_g_hat.squeeze() + audio = audio * MAX_WAV_VALUE + audio_denoised = denoiser(audio.view(1, -1), strength=35)[:, 0] + audio_np = ( + audio_denoised.detach().cpu().numpy().reshape(-1).astype(np.int16) + ) + + # Auto-tuning + if "pc" in pitch_options and "dra" not in pitch_options: + _, output_freq, _, _ = crepe.predict(audio_np, 22050, viterbi=True) + output_pitch = torch.from_numpy(output_freq.astype(np.float32)) + target_pitch = torch.FloatTensor(f0s_wo_silence) + factor = torch.mean(output_pitch) / torch.mean(target_pitch) + + octaves = [0.125, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0] + nearest_octave = min(octaves, key=lambda x: abs(x - factor)) + target_pitch *= nearest_octave + if len(target_pitch) < len(output_pitch): + target_pitch = torch.nn.functional.pad( + target_pitch, + (0, list(output_pitch.shape)[0] - list(target_pitch.shape)[0]), + "constant", + 0, + ) + if len(target_pitch) > len(output_pitch): + target_pitch = target_pitch[0 : list(output_pitch.shape)[0]] + + audio_np = psola.vocode( + audio_np, 22050, target_pitch=target_pitch + ).astype(np.float32) + normalize = (1.0 / np.max(np.abs(audio_np))) ** 0.9 + audio_np = audio_np * normalize * MAX_WAV_VALUE + audio_np = audio_np.astype(np.int16) + + # Resample to 32k + wave = resampy.resample( + audio_np, + h.sampling_rate, + h2.sampling_rate, + filter="sinc_window", + window=scipy.signal.windows.hann, + num_zeros=8, + ) + wave_out = wave.astype(np.int16) + + # HiFi-GAN super-resolution + wave = wave / MAX_WAV_VALUE + wave = torch.FloatTensor(wave).to(torch.device(DEVICE)) + new_mel = mel_spectrogram( + wave.unsqueeze(0), + h2.n_fft, + h2.num_mels, + h2.sampling_rate, + h2.hop_size, + h2.win_size, + h2.fmin, + h2.fmax, + ) + y_g_hat2 = hifigan_sr(new_mel) + audio2 = y_g_hat2.squeeze() + audio2 = audio2 * MAX_WAV_VALUE + audio2_denoised = denoiser(audio2.view(1, -1), strength=35)[:, 0] + + # High-pass filter, mixing and denormalizing + audio2_denoised = audio2_denoised.detach().cpu().numpy().reshape(-1) + b = scipy.signal.firwin( + 101, cutoff=10500, fs=h2.sampling_rate, pass_zero=False + ) + y = scipy.signal.lfilter(b, [1.0], audio2_denoised) + y *= 4.0 # superres strength + y_out = y.astype(np.int16) + y_padded = np.zeros(wave_out.shape) + y_padded[: y_out.shape[0]] = y_out + sr_mix = wave_out + y_padded + + buffer = io.BytesIO() + wavfile.write(buffer, 32000, sr_mix.astype(np.int16)) + b64 = base64.b64encode(buffer.getvalue()) + sound = "data:audio/x-wav;base64," + b64.decode("ascii") + + output_name = "TalkNet_" + str(int(time.time())) + return [sound, arpa, playback_style, output_name] + except Exception: + return [ + None, + str(traceback.format_exc()), + playback_hide, + None, + ] + + +if __name__ == "__main__": + app.run_server( + mode="external", + debug=True, + dev_tools_ui=True, + dev_tools_hot_reload=True, + threaded=True, + ) diff --git a/models/tacotron2/ControllableTalkNet/diffsvc_gui.py b/models/tacotron2/ControllableTalkNet/diffsvc_gui.py new file mode 100644 index 0000000000..837bd4e759 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/diffsvc_gui.py @@ -0,0 +1,368 @@ +import sys +import os +import io +import base64 +import dash +from jupyter_dash import JupyterDash +import dash_core_components as dcc +import dash_html_components as html +from dash.exceptions import PreventUpdate +import gdown +import traceback +from scipy.io import wavfile +import numpy as np +import torch + +sys.path.append("DiffSVC_inference_only") +from end2end import load_e2e_diffsvc, endtoend_from_path, write_to_file + +app = JupyterDash(__name__) +UPLOAD_DIRECTORY = "/content" +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" + +app.layout = html.Div( + children=[ + html.H1( + children="DiffSVC", + style={ + "font-family": "EquestriaWebfont", + "color": "#280e5f", + "font-size": "4em", + "text-align": "center", + "margin-top": "0em", + "margin-bottom": "0em", + }, + ), + html.Label("Character selection", htmlFor="speaker-dropdown"), + dcc.Dropdown( + id="speaker-dropdown", + options=[ + { + "label": "Twilight Sparkle", + "value": "Twilight", + }, + { + "label": "Discord", + "value": "Discord", + }, + { + "label": "Pinkie Pie", + "value": "Pinkie", + }, + {"label": "Nancy", "value": "Nancy"}, + ], + value=None, + style={ + "max-width": "90vw", + "width": "20em", + "margin-bottom": "0.7em", + }, + ), + html.Label( + "Upload reference audio to " + UPLOAD_DIRECTORY, + htmlFor="reference-dropdown", + ), + dcc.Store(id="current-f0s"), + dcc.Store(id="current-filename"), + dcc.Loading( + id="audio-loading", + children=[ + html.Div( + [ + html.Button( + "Update file list", + id="update-button", + style={ + "margin-right": "10px", + }, + ), + dcc.Dropdown( + id="reference-dropdown", + options=[], + value=None, + style={ + "max-width": "80vw", + "width": "30em", + }, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "row", + "margin-left": "50px", + "vertical-align": "middle", + }, + ), + html.Div( + id="audio-loading-output", + style={ + "font-style": "italic", + "margin-bottom": "0.7em", + "text-align": "center", + }, + ), + ], + type="default", + ), + html.Div( + [ + dcc.Checklist( + id="svc-options", + options=[ + {"label": "Singing mode", "value": "no_pc"}, + ], + value=[], + ) + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "row", + "margin-left": "50px", + "margin-bottom": "0.7em", + }, + ), + html.Div( + [ + html.Label("Steps", htmlFor="linsteps-input"), + dcc.Input( + id="linsteps-input", + type="number", + value="1000", + style={"width": "7em", "margin-left": "10px"}, + min=1, + max=1000, + step=1, + ), + html.Label("Step strength", htmlFor="linend-input"), + dcc.Input( + id="linend-input", + type="number", + value="0.06", + style={"width": "7em", "margin-left": "10px"}, + min=0.01, + max=0.3, + step=0.01, + ), + ], + style={ + "display": "grid", + "grid-template-columns": "auto 100px", + "margin-bottom": "0.7em", + }, + ), + dcc.Loading( + html.Div( + [ + html.Button( + "Generate", + id="gen-button", + ), + html.Audio( + id="audio-out", + controls=True, + style={ + "display": "none", + }, + ), + html.Div( + id="generated-info", + style={ + "font-style": "italic", + }, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "column", + }, + ) + ), + html.Footer( + children=""" + Presented by the Pony Preservation Project. Models by Cookie. + """, + style={"margin-top": "2em", "font-size": "0.7em"}, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "column", + "background-color": "#FFF", + }, +) + + +playback_style = { + "margin-top": "0.3em", + "margin-bottom": "0.3em", + "display": "block", + "width": "600px", + "max-width": "90vw", +} + +playback_hide = { + "display": "none", +} + + +@app.callback( + dash.dependencies.Output("reference-dropdown", "options"), + [ + dash.dependencies.Input("update-button", "n_clicks"), + ], +) +def update_filelist(n_clicks): + filelist = [] + supported_formats = [".wav", ".ogg", ".mp3", "flac"] + for x in os.listdir(UPLOAD_DIRECTORY): + if x[-4:].lower() in supported_formats: + filelist.append({"label": x, "value": x}) + return filelist + + +@app.callback( + [ + dash.dependencies.Output("audio-loading-output", "children"), + dash.dependencies.Output("current-filename", "data"), + ], + [ + dash.dependencies.Input("reference-dropdown", "value"), + ], +) +def select_file(dropdown_value): + if dropdown_value is not None: + return [ + "Selected " + dropdown_value, + dropdown_value, + ] + else: + return ["No audio selected", None] + + +def download_model(drive_id, outname): + d = "https://drive.google.com/uc?id=" + model_dir = os.path.join(UPLOAD_DIRECTORY, "models") + out_fullpath = os.path.join(model_dir, outname) + out_dir = os.path.dirname(out_fullpath) + + if not os.path.exists(model_dir): + os.mkdir(model_dir) + if not os.path.exists(out_fullpath): + os.makedirs(out_dir, exist_ok=True) + gdown.download( + d + drive_id, + out_fullpath, + quiet=False, + ) + return out_fullpath + + +svc_model, svc_name, last_end, last_step = None, None, None, None + + +@app.callback( + [ + dash.dependencies.Output("audio-out", "src"), + dash.dependencies.Output("generated-info", "children"), + dash.dependencies.Output("audio-out", "style"), + ], + [dash.dependencies.Input("gen-button", "n_clicks")], + [ + dash.dependencies.State("speaker-dropdown", "value"), + dash.dependencies.State("svc-options", "value"), + dash.dependencies.State("current-filename", "data"), + dash.dependencies.State("linend-input", "value"), + dash.dependencies.State("linsteps-input", "value"), + ], +) +def generate_audio(n_clicks, speaker, svc_options, wav_name, lin_end, lin_n_steps): + global svc_model, svc_name, last_end, last_step + + if n_clicks is None: + raise PreventUpdate + if speaker is None: + return [None, "No character selected", playback_hide] + if wav_name is None: + return [None, "No reference audio selected", playback_hide] + + try: + diffsvc_id = "1Uh17L1JtynFDgX9X6jKLPtD_t4IcTEp9" + asr_id = "1qt0pGhCbH0TltFFwSP2Oy3NClthcpr7N" + hifigan_id = "1QQT0HjMhGgDuPhyesZYbvck2lEY7GHAB" + hifigan_config_id = "1mi_O54zi6nW2eU6VuAO_Y4gv_Df-H9Yv" + + diffsvc_path = download_model( + diffsvc_id, + os.path.join(diffsvc_id, "diffsvc_model"), + ) + asr_path = download_model( + asr_id, + os.path.join(asr_id, "asr_model"), + ) + hifigan_path = download_model( + hifigan_id, + os.path.join(hifigan_id, "hifigan_model"), + ) + download_model( + hifigan_config_id, + os.path.join(hifigan_id, "config.json"), + ) + if svc_name != diffsvc_id: + svc_model = load_e2e_diffsvc( + diffsvc_path=diffsvc_path, + dilated_asr_path=asr_path, + hifigan_path=hifigan_path, + device=DEVICE, + ) + svc_name = diffsvc_id + if last_end != lin_end or last_step != lin_n_steps: + svc_model[0].generator.diffusion.set_noise_schedule( + 1e-4, float(lin_end), int(lin_n_steps), device=DEVICE + ) + last_end = lin_end + last_step = lin_n_steps + + pred_audio = endtoend_from_path( + *svc_model, + os.path.join(UPLOAD_DIRECTORY, wav_name), + speaker, + "no_pc" not in svc_options, + "no_pc" not in svc_options, + "no_pc" not in svc_options, + t_max_step=int(lin_n_steps), + ) + pred_audio *= 32768.0 + + buffer = io.BytesIO() + wavfile.write( + buffer, + svc_model[4].sampling_rate, + pred_audio.squeeze().cpu().numpy().astype(np.int16), + ) + b64 = base64.b64encode(buffer.getvalue()) + sound = "data:audio/x-wav;base64," + b64.decode("ascii") + return [sound, "Conversion complete", playback_style] + except Exception: + return [None, str(traceback.format_exc()), playback_hide] + + +if __name__ == "__main__": + app.run_server( + mode="external", + debug=True, + dev_tools_ui=True, + dev_tools_hot_reload=True, + threaded=True, + ) diff --git a/models/tacotron2/ControllableTalkNet/model_lists/sortanon_models.json b/models/tacotron2/ControllableTalkNet/model_lists/sortanon_models.json new file mode 100644 index 0000000000..3d7033c307 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/model_lists/sortanon_models.json @@ -0,0 +1,165 @@ +[ + {"source": "MLP", + "characters": [ + { + "name": "Applejack", + "drive_id": "1kpEjZ3YqMN3chKSXODOqayEm581rxj4r", + "is_singing": false + }, + { + "name": "Celestia", + "drive_id": "1whXXcnXu9XPcI60xIkTEofpaDYDOw5yB", + "is_singing": false + }, + { + "name": "Chrysalis", + "drive_id": "1bb5jKAcQcEQbx1feVwT1UmAEgocINh-E", + "is_singing": false + }, + { + "name": "Discord", + "drive_id": "1Cg9Oc_K9UDe5WgVDAcaCSbbBoo-Npj1E", + "is_singing": false + }, + { + "name": "Fluttershy", + "drive_id": "1KgVnjrnxZTXgjnI56ilkq5G4UJCbbwZZ", + "is_singing": false + }, + { + "name": "Luna", + "drive_id": "1_ztAbe5YArCMwyyQ_G9lUiz74ym5xJKC", + "is_singing": false + }, + { + "name": "Maud Pie", + "drive_id": "132G6oD0HHPPn4t1H6IkYv18_F0UVLWgi", + "is_singing": false + }, + { + "name": "Pinkie Pie", + "drive_id": "1CdYZ2r52mtgJsFs88U0ZViMSnzpQ_HRp", + "is_singing": false + }, + { + "name": "Rainbow Dash", + "drive_id": "1k3EMXxLC0fLvfxzGbeP6B6plgu9hqCSx", + "is_singing": false + }, + { + "name": "Rarity", + "drive_id": "1QWBvQSso4guc1LRUD40WRJ8DY2CfqHGK", + "is_singing": false + }, + { + "name": "Spike", + "drive_id": "1TKFdmFLttjjzByj2fZW8J70ZHjR-RTwc", + "is_singing": false + }, + { + "name": "Starlight Glimmer", + "drive_id": "1M1AMBq_xjwGTNzRUCXtSLIDJHbcSs3zR", + "is_singing": false + }, + { + "name": "Sunset Shimmer", + "drive_id": "1x1aJt06lBvzUWRlxJ9CEKcFHxQxZPpST", + "is_singing": false + }, + { + "name": "Trixie", + "drive_id": "1a3CYt0-oTTSFjxtZvAVMpClTmQteYua5", + "is_singing": false + }, + { + "name": "Twilight Sparkle", + "drive_id": "1QnOliOAmerMUNuo2wXoH-YoainoSjZen", + "is_singing": false + }, + { + "name": "Twilight Sparkle (whispering)", + "drive_id": "14_TUQVirITdyBh9etfNV8KFFhi_PUs30", + "is_singing": false + }, + { + "name": "Zecora", + "drive_id": "1gL0hqqB7952Q1S185moQd_DRCFfIa3_g", + "is_singing": false + }, + { + "name": "Twilight Sparkle (singing)", + "drive_id": "10CENYWV5ugTXZbnsldN6OKR7wkDEe7V7", + "is_singing": true + }, + { + "name": "Pinkie Pie (singing)", + "drive_id": "1lo-8wI8iB3GArrzIDlaNTA1If8WX0OEf", + "is_singing": true + }, + { + "name": "Applejack (singing)", + "drive_id": "1kegd0J2G4nogaX4IOB2O_vd8msaqMx8M", + "is_singing": true + }, + { + "name": "Fluttershy (singing)", + "drive_id": "1BBdTHis91MwnHTt7tD_xtZ-nQ9SgvqD6", + "is_singing": true + }, + { + "name": "Rarity (singing)", + "drive_id": "1EI7npfQeQYuP6b_Gy8iwA5zAg9Yj5MJA", + "is_singing": true + }, + { + "name": "Rainbow Dash (singing)", + "drive_id": "1q4RgCEVJCSaR_1Dh0rxs6JVRJt30hQRK", + "is_singing": true + }, + { + "name": "Apple Bloom", + "drive_id": "1Wm-7gqws0B3j8mtSQa77Tdk3RZwHEwT5", + "is_singing": false + }, + { + "name": "Scootaloo", + "drive_id": "1YkV1VtP1w5XOx3jYYarrCKSzXCB_FLCy", + "is_singing": false + }, + { + "name": "Sweetie Belle", + "drive_id": "1jLX0Py6j8uY93Fjf2l0HOZQYXiShfWUO", + "is_singing": false + }, + { + "name": "Cozy Glow", + "drive_id": "1cUQQ_4KXTXbmH3MqwwyZu77SpMr20HAL", + "is_singing": false + }, + { + "name": "Big McIntosh", + "drive_id": "1UzL5vo_ykmRdbi-5JOlsQzNKkoV4tJBP", + "is_singing": false + }, + { + "name": "Cadance", + "drive_id": "1w98tMJPfGhWvl797gImMfErPXDz_hA-6", + "is_singing": false + }, + { + "name": "Granny Smith", + "drive_id": "1FJuxLB6MhcHLtmEnOIxhN-yxJ85kmacu", + "is_singing": false + }, + { + "name": "Shining Armor", + "drive_id": "1PUUS71w2ik0uuycNB30nXFze8C7O8OzY", + "is_singing": false + }, + { + "name": "Tirek", + "drive_id": "1rcPDqgDeCIHGDdvfOo-fxfA1XeM4g3CB", + "is_singing": false + } + ]} +] diff --git a/models/tacotron2/ControllableTalkNet/mycroft_talknet.py b/models/tacotron2/ControllableTalkNet/mycroft_talknet.py new file mode 100644 index 0000000000..d94eb8ed04 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/mycroft_talknet.py @@ -0,0 +1,114 @@ +import os +from flask import Flask, request, send_file +import sys +import torch +import numpy as np +from scipy.io import wavfile +import io +from nemo.collections.tts.models import TalkNetSpectModel +from nemo.collections.tts.models import TalkNetPitchModel +from nemo.collections.tts.models import TalkNetDursModel +import json + +sys.path.append("hifi-gan") +from env import AttrDict +from meldataset import MAX_WAV_VALUE +from models import Generator +from denoiser import Denoiser + + +app = Flask(__name__) +RUN_PATH = os.path.dirname(os.path.realpath(__file__)) +DEVICE = "cuda:0" + + +def load_hifigan(model_name, conf_name): + # Load HiFi-GAN + conf = os.path.join("hifi-gan", conf_name + ".json") + with open(conf) as f: + json_config = json.loads(f.read()) + h = AttrDict(json_config) + torch.manual_seed(h.seed) + hifigan = Generator(h).to(torch.device(DEVICE)) + state_dict_g = torch.load(model_name, map_location=torch.device(DEVICE)) + hifigan.load_state_dict(state_dict_g["generator"]) + hifigan.eval() + hifigan.remove_weight_norm() + denoiser = Denoiser(hifigan, mode="normal") + return hifigan, h, denoiser + + +def generate_json(input, outpath): + output = "" + sample_rate = 22050 + lpath = input.split("|")[0].strip() + size = os.stat(lpath).st_size + x = { + "audio_filepath": lpath, + "duration": size / (sample_rate * 2), + "text": input.split("|")[1].strip(), + } + output += json.dumps(x) + "\n" + with open(outpath, "w", encoding="utf8") as w: + w.write(output) + + +def load_talknet(talknet_path): + with torch.no_grad(): + tnmodel = TalkNetSpectModel.restore_from(talknet_path) + durs_path = os.path.join(os.path.dirname(talknet_path), "TalkNetDurs.nemo") + tndurs = TalkNetDursModel.restore_from(durs_path) + tnmodel.add_module("_durs_model", tndurs) + pitch_path = os.path.join(os.path.dirname(talknet_path), "TalkNetPitch.nemo") + tnpitch = TalkNetPitchModel.restore_from(pitch_path) + tnmodel.add_module("_pitch_model", tnpitch) + tnmodel.eval() + return tnmodel + + +def generate_audio(transcript, tnmodel, hifigan, denoiser): + with torch.no_grad(): + tokens = tnmodel.parse(text=transcript.strip()) + spect = tnmodel.generate_spectrogram(tokens=tokens) + + y_g_hat = hifigan(spect.float()) + audio = y_g_hat.squeeze() + audio = audio * MAX_WAV_VALUE + audio_denoised = denoiser(audio.view(1, -1), strength=35)[:, 0] + audio_np = audio_denoised.detach().cpu().numpy().reshape(-1).astype(np.int16) + + buffer = io.BytesIO() + wavfile.write(buffer, 22050, audio_np) + return buffer + + +@app.route("/", methods=["GET"]) +def get_check(): + return "TalkNet server online" + + +@app.route("/api/tts", methods=["GET"]) +def get_tts(): + if "text" not in request.args: + return "" + transcript = request.args.get("text") + return send_file( + generate_audio(transcript, tnmodel, hifigan, denoiser), + attachment_filename="audio.wav", + mimetype="audio/x-wav", + ) + + +if __name__ == "__main__": + hifigan, h, denoiser = load_hifigan( + os.path.join( + RUN_PATH, "models", "1QnOliOAmerMUNuo2wXoH-YoainoSjZen", "hifiganmodel" + ), + "config_v1", + ) + tnmodel = load_talknet( + os.path.join( + RUN_PATH, "models", "1QnOliOAmerMUNuo2wXoH-YoainoSjZen", "TalkNetSpect.nemo" + ) + ) + app.run(debug=False) diff --git a/models/tacotron2/ControllableTalkNet/requirements.txt b/models/tacotron2/ControllableTalkNet/requirements.txt new file mode 100644 index 0000000000..f7270b68cb --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/requirements.txt @@ -0,0 +1,28 @@ +tensorflow==2.4.1 +tensorboard==2.4.1 +dash==1.21.0 +dash-bootstrap-components==0.13.0 +jupyter-dash==0.4.0 +psola==0.0.1 +wget==3.2 +unidecode==1.2.0 +pysptk==0.1.18 +frozendict==2.0.3 +torch==1.8.1+cu111 +torchvision==0.9.1+cu111 +torchaudio==0.8.1 +torchtext==0.9.1 +torch_stft==0.1.4 +kaldiio==2.17.2 +pydub==0.25.1 +pyannote.audio==1.1.2 +g2p_en==2.1.0 +pesq==0.0.2 +pystoi==0.3.3 +crepe==0.0.12 +resampy==0.2.2 +ffmpeg-python==0.2.0 +tqdm +gdown==3.13.0 +editdistance==0.5.3 +ipywidgets==7.6.3 diff --git a/models/tacotron2/ControllableTalkNet/talknet_offline.py b/models/tacotron2/ControllableTalkNet/talknet_offline.py new file mode 100644 index 0000000000..b2b37a6499 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/talknet_offline.py @@ -0,0 +1,17 @@ +if __name__ == "__main__": + import os + from controllable_talknet import * + + if os.path.exists("/talknet/is_docker"): + app.run_server( + host="0.0.0.0", + mode="external", + debug=False, + threaded=True, + ) + else: + app.run_server( + mode="external", + debug=False, + threaded=True, + ) diff --git a/models/tacotron2/ControllableTalkNet/talknet_singer.py b/models/tacotron2/ControllableTalkNet/talknet_singer.py new file mode 100644 index 0000000000..34f573dca7 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/talknet_singer.py @@ -0,0 +1,225 @@ +from collections import OrderedDict +from typing import List + +import torch +from hydra.utils import instantiate +from omegaconf import DictConfig +from pytorch_lightning import Trainer +from torch import nn +from torch.nn import functional as F +import numpy as np + +from nemo.collections.asr.data.audio_to_text import AudioToCharWithDursF0Dataset +from nemo.collections.tts.helpers.helpers import get_mask_from_lengths +from nemo.collections.tts.models.base import SpectrogramGenerator +from nemo.collections.tts.modules.talknet import ( + GaussianEmbedding, + MaskedInstanceNorm1d, + StyleResidual, +) +from nemo.core import Exportable +from nemo.core.classes import ModelPT, PretrainedModelInfo, typecheck +from nemo.core.neural_types import MelSpectrogramType, NeuralType + + +class TalkNetSingerModel(SpectrogramGenerator, Exportable): + """TalkNet's mel spectrogram prediction pipeline.""" + + @property + def output_types(self): + return OrderedDict( + {"mel-spectrogram": NeuralType(("B", "D", "T"), MelSpectrogramType())} + ) + + def __init__(self, cfg: DictConfig, trainer: "Trainer" = None): + super().__init__(cfg=cfg, trainer=trainer) + typecheck.set_typecheck_enabled(enabled=False) + + cfg = self._cfg + self.vocab = AudioToCharWithDursF0Dataset.make_vocab( + **cfg.train_ds.dataset.vocab + ) + self.blanking = cfg.train_ds.dataset.blanking + self.preprocessor = instantiate(cfg.preprocessor) + self.embed = GaussianEmbedding(self.vocab, cfg.d_char) + self.norm_f0 = MaskedInstanceNorm1d(1) + self.res_f0 = StyleResidual(cfg.d_char, 1, kernel_size=3) + self.model = instantiate(cfg.model) + d_out = cfg.model.jasper[-1].filters + self.proj = nn.Conv1d(d_out, cfg.n_mels, kernel_size=1) + + def forward(self, text, text_len, durs, f0): + x, x_len = self.embed(text, durs).transpose(1, 2), durs.sum(-1) + f0_sin = np.nan_to_num(np.sin(9.06472 * np.log(0.0363636 * f0.cpu()))) + f0_cos = np.nan_to_num(np.cos(9.06472 * np.log(0.0363636 * f0.cpu()))) + f0_note = torch.stack( + (torch.from_numpy(f0_sin), torch.from_numpy(f0_cos)), 1 + ).to(self.device) + f0, f0_mask = f0.clone(), f0 > 0.0 + f0 = self.norm_f0(f0.unsqueeze(1), f0_mask) + f0[~f0_mask.unsqueeze(1)] = 0.0 + x = self.res_f0(x, f0) + x = torch.cat((x, f0_note), 1) + y, _ = self.model(x, x_len) + mel = self.proj(y) + return mel + + @staticmethod + def _metrics(true_mel, true_mel_len, pred_mel): + loss = F.mse_loss(pred_mel, true_mel, reduction="none").mean(dim=-2) + mask = get_mask_from_lengths(true_mel_len) + loss *= mask.float() + loss = loss.sum() / mask.sum() + return loss + + def training_step(self, batch, batch_idx): + audio, audio_len, text, text_len, durs, f0, f0_mask = batch + mel, mel_len = self.preprocessor(audio, audio_len) + pred_mel = self(text=text, text_len=text_len, durs=durs, f0=f0) + loss = self._metrics(true_mel=mel, true_mel_len=mel_len, pred_mel=pred_mel) + train_log = {"train_loss": loss} + return {"loss": loss, "progress_bar": train_log, "log": train_log} + + def validation_step(self, batch, batch_idx): + audio, audio_len, text, text_len, durs, f0, f0_mask = batch + mel, mel_len = self.preprocessor(audio, audio_len) + pred_mel = self(text=text, text_len=text_len, durs=durs, f0=f0) + loss = self._metrics(true_mel=mel, true_mel_len=mel_len, pred_mel=pred_mel) + val_log = {"val_loss": loss} + self.log_dict( + val_log, prog_bar=False, on_epoch=True, logger=True, sync_dist=True + ) + + @staticmethod + def _loader(cfg): + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader( # noqa + dataset=dataset, + collate_fn=dataset.collate_fn, + **cfg.dataloader_params, + ) + + def setup_training_data(self, cfg): + self._train_dl = self._loader(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self._loader(cfg) + + def setup_test_data(self, cfg): + """Omitted.""" + pass + + def parse(self, text: str, **kwargs) -> torch.Tensor: + return torch.tensor(self.vocab.encode(text)).long().unsqueeze(0).to(self.device) + + def generate_spectrogram(self, tokens: torch.Tensor, **kwargs) -> torch.Tensor: + assert hasattr(self, "_durs_model") and hasattr(self, "_pitch_model") + + if self.blanking: + tokens = [ + AudioToCharWithDursF0Dataset.interleave( + x=torch.empty(len(t) + 1, dtype=torch.long, device=t.device).fill_( + self.vocab.blank + ), + y=t, + ) + for t in tokens + ] + tokens = AudioToCharWithDursF0Dataset.merge( + tokens, value=self.vocab.pad, dtype=torch.long + ) + + text_len = torch.tensor(tokens.shape[-1], dtype=torch.long).unsqueeze(0) + durs = self._durs_model(tokens, text_len) + durs = durs.exp() - 1 + durs[durs < 0.0] = 0.0 + durs = durs.round().long() + + # Pitch + f0_sil, f0_body = self._pitch_model(tokens, text_len, durs) + sil_mask = f0_sil.sigmoid() > 0.5 + f0 = f0_body * self._pitch_model.f0_std + self._pitch_model.f0_mean + f0 = (~sil_mask * f0).float() + + # Spect + mel = self(tokens, text_len, durs, f0) + + return mel + + def forward_for_export(self, tokens: torch.Tensor, text_len: torch.Tensor): + durs = self._durs_model(tokens, text_len) + durs = durs.exp() - 1 + durs[durs < 0.0] = 0.0 + durs = durs.round().long() + + # Pitch + f0_sil, f0_body = self._pitch_model(tokens, text_len, durs) + sil_mask = f0_sil.sigmoid() > 0.5 + f0 = f0_body * self._pitch_model.f0_std + self._pitch_model.f0_mean + f0 = (~sil_mask * f0).float() + + # Spect + x, x_len = self.embed(tokens, durs).transpose(1, 2), durs.sum(-1) + f0_sin = np.nan_to_num(np.sin(9.06472 * np.log(0.0363636 * f0.cpu()))) + f0_cos = np.nan_to_num(np.cos(9.06472 * np.log(0.0363636 * f0.cpu()))) + f0_note = torch.stack( + (torch.from_numpy(f0_sin), torch.from_numpy(f0_cos)), 1 + ).to(self.device) + f0, f0_mask = f0.clone(), f0 > 0.0 + f0 = self.norm_f0(f0.unsqueeze(1), f0_mask) + f0[~f0_mask.unsqueeze(1)] = 0.0 + x = self.res_f0(x, f0) + x = torch.cat((x, f0_note), 1) + y, _ = self.model(x, x_len) + mel = self.proj(y) + + return mel + + def force_spectrogram( + self, tokens: torch.Tensor, durs: torch.Tensor, f0: torch.Tensor, **kwargs + ) -> torch.Tensor: + if self.blanking: + tokens = [ + AudioToCharWithDursF0Dataset.interleave( + x=torch.empty(len(t) + 1, dtype=torch.long, device=t.device).fill_( + self.vocab.blank + ), + y=t, + ) + for t in tokens + ] + tokens = AudioToCharWithDursF0Dataset.merge( + tokens, value=self.vocab.pad, dtype=torch.long + ) + + text_len = torch.tensor(tokens.shape[-1], dtype=torch.long).unsqueeze(0) + durs_len = torch.tensor(durs.shape[-1], dtype=torch.long).unsqueeze(0) + assert text_len == durs_len + + # Spect + mel = self(tokens, text_len, durs, f0) + return mel + + @classmethod + def list_available_models(cls) -> "List[PretrainedModelInfo]": + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_en_talknet", + location=( + "https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_talknet/versions/1.0.0rc1/files" + "/talknet_spect.nemo" + ), + description=( + "This model is trained on LJSpeech sampled at 22050Hz, and can be used to generate female " + "English voices with an American accent." + ), + class_=cls, # noqa + aliases=["TalkNet-22050Hz"], + ) + list_of_models.append(model) + return list_of_models