diff --git a/models/tacotron2/ControllableTalkNet/Dockerfile b/models/tacotron2/ControllableTalkNet/Dockerfile new file mode 100644 index 0000000000..be6e6b1481 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/Dockerfile @@ -0,0 +1,29 @@ +FROM nvidia/cuda:11.0.3-base-ubuntu20.04 +ENV TZ=Etc/GMT +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN apt-get update && apt-get install -y git ffmpeg python3.8 python3-pip curl +RUN git clone -q https://github.com/SortAnon/ControllableTalkNet /talknet +RUN git clone -q https://github.com/SortAnon/hifi-gan /talknet/hifi-gan +RUN python3.8 -m pip --no-cache-dir install -r "/talknet/requirements.txt" -f https://download.pytorch.org/whl/torch_stable.html +RUN python3.8 -m pip --no-cache-dir install git+https://github.com/SortAnon/NeMo.git +RUN python3.8 -m pip uninstall -y pesq +RUN python3.8 -m pip install pesq==0.0.2 +RUN touch /talknet/is_docker +WORKDIR /talknet +EXPOSE 8050 + +RUN printf "#!/bin/bash \ + \necho Updating TalkNet... \ + \ngit -C /talknet reset --hard origin/main -q \ + \ngit -C /talknet pull origin main -q \ + \necho Updating HiFi-GAN... \ + \ngit -C /talknet reset --hard origin/main -q \ + \ngit -C /talknet pull origin main -q \ + \necho Updating Python dependencies... \ + \npython3.8 -m pip --quiet --no-cache-dir install -r /talknet/requirements.txt -f https://download.pytorch.org/whl/torch_stable.html \ + \npython3.8 -m pip --quiet --no-cache-dir install git+https://github.com/SortAnon/NeMo.git \ + \necho Launching TalkNet... \ + \npython3.8 talknet_offline.py\n" > /talknet/docker_launch.sh + +RUN chmod +x /talknet/docker_launch.sh +CMD ["./docker_launch.sh"] diff --git a/models/tacotron2/ControllableTalkNet/LICENSE b/models/tacotron2/ControllableTalkNet/LICENSE new file mode 100644 index 0000000000..0e259d42c9 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/LICENSE @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/models/tacotron2/ControllableTalkNet/README.md b/models/tacotron2/ControllableTalkNet/README.md new file mode 100644 index 0000000000..20f0d0429a --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/README.md @@ -0,0 +1,26 @@ +# Controllable TalkNet +Controllable TalkNet is a web application that lets you synthesize speech, +which mimics the pitch and pacing of an existing audio clip. It's based on [NVIDIA's implementation](https://github.com/NVIDIA/NeMo) +of TalkNet 2, with some changes to support singing synthesis and higher audio quality. + +## Requirements +* A Google account to run Colab, or... +* An NVIDIA GPU with 4+ GB of VRAM +* 10 GB of free space + +## How to run +### Google Colab +* [Go to the Colab notebook](https://colab.research.google.com/drive/1aj6Jk8cpRw7SsN3JSYCv57CrR6s0gYPB) and follow the instructions. + +### TalkNet Offline (Windows) +* [Download the setup script](https://github.com/SortAnon/ControllableTalkNet/releases/latest/download/TalkNetOffline.zip) +and extract it to a folder. +* Run setup.bat. The initial setup will take about 20 minutes. +* When it's done, run talknet.bat to start TalkNet on http://127.0.0.1:8050/. To download updates, run update.bat. + +### Docker (Linux) +* Install Docker and NVIDIA Container Toolkit. +* [Download the Dockerfile.](https://raw.githubusercontent.com/SortAnon/ControllableTalkNet/main/Dockerfile) +Open a terminal, and navigate to the directory where you saved it. +* Run ```docker build -t talknet-offline .``` to build the image. Add ```sudo``` if you're not using rootless Docker. +* Run ```docker run -it --gpus all -p 8050:8050 talknet-offline``` to start TalkNet on http://127.0.0.1:8050/. diff --git a/models/tacotron2/ControllableTalkNet/TalkNet_Training_Offline.ipynb b/models/tacotron2/ControllableTalkNet/TalkNet_Training_Offline.ipynb new file mode 100644 index 0000000000..44b8cda03f --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/TalkNet_Training_Offline.ipynb @@ -0,0 +1,925 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# TalkNet Training (offline)\n", + "\n", + "To train a 22KHz TalkNet, run the cells below and follow the instructions.\n", + "\n", + "The notebook will automatically resume training any models from the last saved checkpoint. If you're resuming from a new session, always re-run step 1 first." + ], + "metadata": { + "id": "Gss5Ox_RNiba" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 1:** Configure training data paths. Create the following and change the paths below:\n", + "* A dataset of .wav files, packaged as a .zip or .tar file\n", + "* Training and validation filelists, in LJSpeech format with relative paths (note: ARPABET transcripts are not supported)\n", + "* An output path for checkpoints" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "### CHANGE THESE ###\n", + "\n", + "dataset = \"/absolute/path_to_dataset.zip\"\n", + "train_filelist = \"/absolute/train_filelist.txt\"\n", + "val_filelist = \"/absolute/val_filelist.txt\"\n", + "output_dir = \"/absolute/name_of_character\"\n", + "\n", + "### ------------ ###\n", + "\n", + "import os\n", + "import gdown\n", + "import platform\n", + "\n", + "assert os.path.exists(dataset), \"Cannot find dataset\"\n", + "assert os.path.exists(train_filelist), \"Cannot find training filelist\"\n", + "assert os.path.exists(val_filelist), \"Cannot find validation filelist\"\n", + "if not os.path.exists(output_dir):\n", + " os.makedirs(output_dir)\n", + "\n", + "if os.getcwd() != output_dir:\n", + " cwd = os.getcwd()\n", + "else:\n", + " os.chdir(cwd)\n", + " \n", + "# Download pre-trained models\n", + "if not os.path.exists(\"talknet_spect.nemo\"):\n", + " print(\"Downloading pre-trained models...\")\n", + " zip_path = \"TalkNet_Pretrained.zip\"\n", + " if not os.path.exists(zip_path) or os.stat(zip_path).st_size < 100:\n", + " d = 'https://drive.google.com/uc?id='\n", + " gdown.download(d+\"19wSym9mNEnmzLS9XdPlfNAW9_u-mP1hR\", zip_path, quiet=False)\n", + " print(platform.system())\n", + " if \"Windows\" in platform.system():\n", + " !tar -xf {zip_path}\n", + " else:\n", + " !unzip -qo {zip_path}\n", + " os.remove(zip_path)\n", + "\n", + "print(\"OK\")" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "nfSawDUD5tqv" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 2:** Dataset processing, part 1.\n", + "\n", + "If this step fails, try the following:\n", + "* Make sure your filelists are correct. They should have relative \n", + "paths that match the contents of the archive." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "import os\n", + "import shutil\n", + "import sys\n", + "import json\n", + "import nemo\n", + "import torch\n", + "import torchaudio\n", + "import numpy as np\n", + "#from pysptk import sptk\n", + "from pathlib import Path\n", + "from tqdm.notebook import tqdm\n", + "import ffmpeg\n", + "\n", + "def fix_transcripts(inpath):\n", + " found_arpabet = False\n", + " found_grapheme = False\n", + " with open(inpath, \"r\", encoding=\"utf8\") as f:\n", + " lines = f.readlines()\n", + " with open(inpath, \"w\", encoding=\"utf8\") as f:\n", + " for l in lines:\n", + " if l.strip() == \"\":\n", + " continue\n", + " if \"{\" in l:\n", + " if not found_arpabet:\n", + " print(\"Warning: Skipping ARPABET lines (not supported).\")\n", + " found_arpabet = True\n", + " else:\n", + " f.write(l)\n", + " found_grapheme = True\n", + " assert found_grapheme, \"No non-ARPABET lines found in \" + inpath\n", + "\n", + "def generate_json(inpath, outpath):\n", + " output = \"\"\n", + " sample_rate = 22050\n", + " with open(inpath, \"r\", encoding=\"utf8\") as f:\n", + " for l in f.readlines():\n", + " lpath = l.split(\"|\")[0].strip()\n", + " if lpath[:5] != \"wavs/\":\n", + " lpath = \"wavs/\" + lpath\n", + " lpath = os.path.join(output_dir, \"_training_data\", lpath)\n", + " size = os.stat(lpath).st_size\n", + " x = {\n", + " \"audio_filepath\": lpath,\n", + " \"duration\": size / (sample_rate * 2),\n", + " \"text\": l.split(\"|\")[1].strip(),\n", + " }\n", + " output += json.dumps(x) + \"\\n\"\n", + " with open(outpath, \"w\", encoding=\"utf8\") as w:\n", + " w.write(output)\n", + "\n", + "def convert_to_22k(inpath):\n", + " if inpath.strip()[-4:].lower() != \".wav\":\n", + " print(\"Warning: \" + inpath.strip() + \" is not a .wav file!\")\n", + " return\n", + " ffmpeg.input(inpath).output(\n", + " inpath + \"_22k.wav\",\n", + " ar=\"22050\",\n", + " ac=\"1\",\n", + " acodec=\"pcm_s16le\",\n", + " map_metadata=\"-1\",\n", + " fflags=\"+bitexact\",\n", + " ).overwrite_output().run(quiet=True)\n", + " os.remove(inpath)\n", + " os.rename(inpath + \"_22k.wav\", inpath)\n", + "\n", + "# Extract dataset\n", + "os.chdir(output_dir)\n", + "\n", + "if os.path.exists(\"_training_data\"):\n", + " shutil.rmtree(\"_training_data\")\n", + "os.mkdir(\"_training_data\")\n", + "os.chdir(\"_training_data\")\n", + "os.mkdir(\"wavs\")\n", + "os.chdir(\"wavs\")\n", + "if dataset[-4:] == \".zip\":\n", + " if \"Windows\" in platform.system():\n", + " !tar -xf \"{dataset}\"\n", + " else:\n", + " !unzip -qo \"{dataset}\"\n", + "elif dataset[-4:] == \".tar\":\n", + " !tar -xf \"{dataset}\"\n", + "else:\n", + " raise Exception(\"Unknown extension for dataset\")\n", + "if os.path.exists(os.path.join(output_dir, \"_training_data\", \"wavs\", \"wavs\")):\n", + " shutil.move(\n", + " os.path.join(output_dir, \"_training_data\", \"wavs\", \"wavs\"), \n", + " os.path.join(output_dir, \"_training_data\", \"tempwavs\")\n", + " )\n", + " shutil.rmtree(os.path.join(output_dir, \"_training_data\", \"wavs\"))\n", + " shutil.move(\n", + " os.path.join(output_dir, \"_training_data\", \"tempwavs\"), \n", + " os.path.join(output_dir, \"_training_data\", \"wavs\")\n", + " )\n", + "# Filelist for preprocessing\n", + "os.chdir(output_dir)\n", + "shutil.copy(train_filelist, \"trainfiles.txt\")\n", + "shutil.copy(val_filelist, \"valfiles.txt\")\n", + "fix_transcripts(\"trainfiles.txt\")\n", + "fix_transcripts(\"valfiles.txt\")\n", + "seen_files = []\n", + "with open(\"trainfiles.txt\") as f:\n", + " t = f.read().split(\"\\n\")\n", + "with open(\"valfiles.txt\") as f:\n", + " v = f.read().split(\"\\n\")\n", + " all_filelist = t[:] + v[:]\n", + "with open(\"allfiles.txt\", \"w\") as f:\n", + " for x in all_filelist:\n", + " if x.strip() == \"\":\n", + " continue\n", + " if x.split(\"|\")[0] not in seen_files:\n", + " seen_files.append(x.split(\"|\")[0])\n", + " f.write(x.strip() + \"\\n\")\n", + "\n", + "# Ensure audio is 22k\n", + "print(\"Converting audio...\")\n", + "for r, _, f in os.walk(os.path.join(\"_training_data\", \"wavs\")):\n", + " for name in tqdm(f):\n", + " convert_to_22k(os.path.join(r, name))\n", + "\n", + "# Convert to JSON\n", + "generate_json(\"trainfiles.txt\", \"trainfiles.json\")\n", + "generate_json(\"valfiles.txt\", \"valfiles.json\")\n", + "generate_json(\"allfiles.txt\", \"allfiles.json\")\n", + "\n", + "print(\"OK\")" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "bxFr3Fdi_kOC" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 3:** Dataset processing, part 2. This takes a while, but\n", + "you only have to run this once per dataset (results are saved to Drive).\n", + "\n", + "If this step fails, try the following:\n", + "* Make sure your dataset only contains WAV files." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "# Extract phoneme duration\n", + "\n", + "import json\n", + "from nemo.collections.asr.models import EncDecCTCModel\n", + "asr_model = EncDecCTCModel.from_pretrained(model_name=\"asr_talknet_aligner\").cpu().eval()\n", + "\n", + "def forward_extractor(tokens, log_probs, blank):\n", + " \"\"\"Computes states f and p.\"\"\"\n", + " n, m = len(tokens), log_probs.shape[0]\n", + " # `f[s, t]` -- max sum of log probs for `s` first codes\n", + " # with `t` first timesteps with ending in `tokens[s]`.\n", + " f = np.empty((n + 1, m + 1), dtype=float)\n", + " f.fill(-(10 ** 9))\n", + " p = np.empty((n + 1, m + 1), dtype=int)\n", + " f[0, 0] = 0.0 # Start\n", + " for s in range(1, n + 1):\n", + " c = tokens[s - 1]\n", + " for t in range((s + 1) // 2, m + 1):\n", + " f[s, t] = log_probs[t - 1, c]\n", + " # Option #1: prev char is equal to current one.\n", + " if s == 1 or c == blank or c == tokens[s - 3]:\n", + " options = f[s : (s - 2 if s > 1 else None) : -1, t - 1]\n", + " else: # Is not equal to current one.\n", + " options = f[s : (s - 3 if s > 2 else None) : -1, t - 1]\n", + " f[s, t] += np.max(options)\n", + " p[s, t] = np.argmax(options)\n", + " return f, p\n", + "\n", + "\n", + "def backward_extractor(f, p):\n", + " \"\"\"Computes durs from f and p.\"\"\"\n", + " n, m = f.shape\n", + " n -= 1\n", + " m -= 1\n", + " durs = np.zeros(n, dtype=int)\n", + " if f[-1, -1] >= f[-2, -1]:\n", + " s, t = n, m\n", + " else:\n", + " s, t = n - 1, m\n", + " while s > 0:\n", + " durs[s - 1] += 1\n", + " s -= p[s, t]\n", + " t -= 1\n", + " assert durs.shape[0] == n\n", + " assert np.sum(durs) == m\n", + " assert np.all(durs[1::2] > 0)\n", + " return durs\n", + "\n", + "def preprocess_tokens(tokens, blank):\n", + " new_tokens = [blank]\n", + " for c in tokens:\n", + " new_tokens.extend([c, blank])\n", + " tokens = new_tokens\n", + " return tokens\n", + "\n", + "data_config = {\n", + " 'manifest_filepath': \"allfiles.json\",\n", + " 'sample_rate': 22050,\n", + " 'labels': asr_model.decoder.vocabulary,\n", + " 'batch_size': 1,\n", + "}\n", + "\n", + "parser = nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset.make_vocab(\n", + " notation='phonemes', punct=True, spaces=True, stresses=False, add_blank_at=\"last\"\n", + ")\n", + "\n", + "dataset = nemo.collections.asr.data.audio_to_text._AudioTextDataset(\n", + " manifest_filepath=data_config['manifest_filepath'], sample_rate=data_config['sample_rate'], parser=parser,\n", + ")\n", + "\n", + "dl = torch.utils.data.DataLoader(\n", + " dataset=dataset, batch_size=data_config['batch_size'], collate_fn=dataset.collate_fn, shuffle=False,\n", + ")\n", + "\n", + "blank_id = asr_model.decoder.num_classes_with_blank - 1\n", + "\n", + "if os.path.exists(os.path.join(output_dir, \"durations.pt\")):\n", + " print(\"durations.pt already exists; skipping\")\n", + "else:\n", + " dur_data = {}\n", + " for sample_idx, test_sample in tqdm(enumerate(dl), total=len(dl)):\n", + " log_probs, _, greedy_predictions = asr_model(\n", + " input_signal=test_sample[0], input_signal_length=test_sample[1]\n", + " )\n", + "\n", + " log_probs = log_probs[0].cpu().detach().numpy()\n", + " seq_ids = test_sample[2][0].cpu().detach().numpy()\n", + "\n", + " target_tokens = preprocess_tokens(seq_ids, blank_id)\n", + "\n", + " f, p = forward_extractor(target_tokens, log_probs, blank_id)\n", + " durs = backward_extractor(f, p)\n", + "\n", + " dur_key = Path(dl.dataset.collection[sample_idx].audio_file).stem\n", + " dur_data[dur_key] = {\n", + " 'blanks': torch.tensor(durs[::2], dtype=torch.long).cpu().detach(), \n", + " 'tokens': torch.tensor(durs[1::2], dtype=torch.long).cpu().detach()\n", + " }\n", + "\n", + " del test_sample\n", + "\n", + " torch.save(dur_data, os.path.join(output_dir, \"durations.pt\"))\n", + "\n", + "#Extract F0 (pitch)\n", + "import crepe\n", + "from scipy.io import wavfile\n", + "\n", + "def crepe_f0(audio_file, hop_length=256):\n", + " sr, audio = wavfile.read(audio_file)\n", + " audio_x = np.arange(0, len(audio)) / 22050.0\n", + " time, frequency, confidence, activation = crepe.predict(audio, sr, viterbi=True)\n", + "\n", + " x = np.arange(0, len(audio), hop_length) / 22050.0\n", + " freq_interp = np.interp(x, time, frequency)\n", + " conf_interp = np.interp(x, time, confidence)\n", + " audio_interp = np.interp(x, audio_x, np.absolute(audio)) / 32768.0\n", + " weights = [0.5, 0.25, 0.25]\n", + " audio_smooth = np.convolve(audio_interp, np.array(weights)[::-1], \"same\")\n", + "\n", + " conf_threshold = 0.25\n", + " audio_threshold = 0.0005\n", + " for i in range(len(freq_interp)):\n", + " if conf_interp[i] < conf_threshold:\n", + " freq_interp[i] = 0.0\n", + " if audio_smooth[i] < audio_threshold:\n", + " freq_interp[i] = 0.0\n", + "\n", + " # Hack to make f0 and mel lengths equal\n", + " if len(audio) % hop_length == 0:\n", + " freq_interp = np.pad(freq_interp, pad_width=[0, 1])\n", + " return torch.from_numpy(freq_interp.astype(np.float32))\n", + "\n", + "if os.path.exists(os.path.join(output_dir, \"f0s.pt\")):\n", + " print(\"f0s.pt already exists; skipping\")\n", + "else:\n", + " f0_data = {}\n", + " with open(\"allfiles.json\") as f:\n", + " for i, l in enumerate(f.readlines()):\n", + " print(str(i))\n", + " audio_path = json.loads(l)[\"audio_filepath\"]\n", + " f0_data[Path(audio_path).stem] = crepe_f0(audio_path)\n", + "\n", + " # calculate f0 stats (mean & std) only for train set\n", + " with open(\"trainfiles.json\") as f:\n", + " train_ids = {Path(json.loads(l)[\"audio_filepath\"]).stem for l in f}\n", + " all_f0 = torch.cat([f0[f0 >= 1e-5] for f0_id, f0 in f0_data.items() if f0_id in train_ids])\n", + "\n", + " F0_MEAN, F0_STD = all_f0.mean().item(), all_f0.std().item() \n", + " print(\"F0_MEAN: \" + str(F0_MEAN) + \", F0_STD: \" + str(F0_STD))\n", + " torch.save(f0_data, os.path.join(output_dir, \"f0s.pt\"))\n", + " with open(os.path.join(output_dir, \"f0_info.json\"), \"w\") as f:\n", + " f.write(json.dumps({\"FO_MEAN\": F0_MEAN, \"F0_STD\": F0_STD}))\n", + "\n", + "print(\"OK\")" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "sos9vsxPkIN7" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 4:** Train duration predictor.\n", + "\n", + "If CUDA runs out of memory, try the following:\n", + "* Click on Kernel -> Restart, re-run step 1, and try again.\n", + "* If that doesn't help, reduce the batch size (default 64)." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "batch_size = 64\n", + "\n", + "import os\n", + "from hydra.experimental import compose, initialize\n", + "from hydra.core.global_hydra import GlobalHydra\n", + "from omegaconf import OmegaConf\n", + "import pytorch_lightning as pl\n", + "from nemo.collections.common.callbacks import LogEpochTimeCallback\n", + "from nemo.collections.tts.models import TalkNetDursModel\n", + "from nemo.core.config import hydra_runner\n", + "from nemo.utils.exp_manager import exp_manager\n", + "\n", + "epochs = 20\n", + "learning_rate = 1e-3\n", + "min_learning_rate = 3e-6\n", + "load_checkpoints = True\n", + "\n", + "def train(cfg):\n", + " cfg.sample_rate = 22050\n", + " cfg.train_dataset = os.path.join(output_dir, \"trainfiles.json\")\n", + " cfg.validation_datasets = os.path.join(output_dir, \"valfiles.json\")\n", + " cfg.durs_file = os.path.join(output_dir, \"durations.pt\")\n", + " cfg.f0_file = os.path.join(output_dir, \"f0s.pt\")\n", + " cfg.trainer.accelerator = \"dp\"\n", + " cfg.trainer.max_epochs = epochs\n", + " cfg.trainer.check_val_every_n_epoch = 5\n", + " cfg.model.train_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.validation_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.optim.lr = learning_rate\n", + " cfg.model.optim.sched.min_lr = min_learning_rate\n", + " cfg.exp_manager.exp_dir = output_dir\n", + "\n", + " # Find checkpoints\n", + " ckpt_path = \"\"\n", + " if load_checkpoints:\n", + " path0 = os.path.join(output_dir, \"TalkNetDurs\")\n", + " if os.path.exists(path0):\n", + " path1 = sorted(os.listdir(path0))\n", + " for i in range(len(path1)):\n", + " path2 = os.path.join(path0, path1[-(1+i)], \"checkpoints\")\n", + " if os.path.exists(path2):\n", + " match = [x for x in os.listdir(path2) if \"last.ckpt\" in x]\n", + " if len(match) > 0:\n", + " ckpt_path = os.path.join(path2, match[0])\n", + " print(\"Resuming training from \" + match[0])\n", + " break\n", + " \n", + " if ckpt_path != \"\":\n", + " trainer = pl.Trainer(**cfg.trainer, resume_from_checkpoint = ckpt_path)\n", + " model = TalkNetDursModel(cfg=cfg.model, trainer=trainer)\n", + " else:\n", + " warmstart_path = os.path.join(cwd, \"talknet_durs.nemo\")\n", + " trainer = pl.Trainer(**cfg.trainer)\n", + " model = TalkNetDursModel.restore_from(warmstart_path, override_config_path=cfg)\n", + " model.set_trainer(trainer)\n", + " model.setup_training_data(cfg.model.train_ds)\n", + " model.setup_validation_data(cfg.model.validation_ds)\n", + " model.setup_optimization(cfg.model.optim)\n", + " print(\"Warm-starting from \" + warmstart_path)\n", + " exp_manager(trainer, cfg.get('exp_manager', None))\n", + " trainer.callbacks.extend([pl.callbacks.LearningRateMonitor(), LogEpochTimeCallback()]) # noqa\n", + " trainer.fit(model)\n", + "\n", + "os.chdir(cwd)\n", + "GlobalHydra().clear()\n", + "initialize(config_path=\"conf\")\n", + "cfg = compose(config_name=\"talknet-durs\")\n", + "train(cfg)\n" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "nM7-bMpKO7U2" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 5:** Train pitch predictor.\n", + "\n", + "If CUDA runs out of memory, try the following:\n", + "* Click on Kernel -> Restart, re-run step 1, and try again.\n", + "* If that doesn't help, reduce the batch size (default 64)." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "batch_size = 64\n", + "epochs = 50\n", + "\n", + "import json\n", + "\n", + "with open(os.path.join(output_dir, \"f0_info.json\"), \"r\") as f:\n", + " f0_info = json.load(f)\n", + " f0_mean = f0_info[\"FO_MEAN\"]\n", + " f0_std = f0_info[\"F0_STD\"]\n", + "\n", + "learning_rate = 1e-3\n", + "min_learning_rate = 3e-6\n", + "load_checkpoints = True\n", + "\n", + "import os\n", + "from hydra.experimental import compose, initialize\n", + "from hydra.core.global_hydra import GlobalHydra\n", + "from omegaconf import OmegaConf\n", + "import pytorch_lightning as pl\n", + "from nemo.collections.common.callbacks import LogEpochTimeCallback\n", + "from nemo.collections.tts.models import TalkNetPitchModel\n", + "from nemo.core.config import hydra_runner\n", + "from nemo.utils.exp_manager import exp_manager\n", + "\n", + "def train(cfg):\n", + " cfg.sample_rate = 22050\n", + " cfg.train_dataset = os.path.join(output_dir, \"trainfiles.json\")\n", + " cfg.validation_datasets = os.path.join(output_dir, \"valfiles.json\")\n", + " cfg.durs_file = os.path.join(output_dir, \"durations.pt\")\n", + " cfg.f0_file = os.path.join(output_dir, \"f0s.pt\")\n", + " cfg.trainer.accelerator = \"dp\"\n", + " cfg.trainer.max_epochs = epochs\n", + " cfg.trainer.check_val_every_n_epoch = 5\n", + " cfg.model.f0_mean=f0_mean\n", + " cfg.model.f0_std=f0_std\n", + " cfg.model.train_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.validation_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.optim.lr = learning_rate\n", + " cfg.model.optim.sched.min_lr = min_learning_rate\n", + " cfg.exp_manager.exp_dir = output_dir\n", + "\n", + " # Find checkpoints\n", + " ckpt_path = \"\"\n", + " if load_checkpoints:\n", + " path0 = os.path.join(output_dir, \"TalkNetPitch\")\n", + " if os.path.exists(path0):\n", + " path1 = sorted(os.listdir(path0))\n", + " for i in range(len(path1)):\n", + " path2 = os.path.join(path0, path1[-(1+i)], \"checkpoints\")\n", + " if os.path.exists(path2):\n", + " match = [x for x in os.listdir(path2) if \"last.ckpt\" in x]\n", + " if len(match) > 0:\n", + " ckpt_path = os.path.join(path2, match[0])\n", + " print(\"Resuming training from \" + match[0])\n", + " break\n", + " \n", + " if ckpt_path != \"\":\n", + " trainer = pl.Trainer(**cfg.trainer, resume_from_checkpoint = ckpt_path)\n", + " model = TalkNetPitchModel(cfg=cfg.model, trainer=trainer)\n", + " else:\n", + " warmstart_path = os.path.join(cwd, \"talknet_pitch.nemo\")\n", + " trainer = pl.Trainer(**cfg.trainer)\n", + " model = TalkNetPitchModel.restore_from(warmstart_path, override_config_path=cfg)\n", + " model.set_trainer(trainer)\n", + " model.setup_training_data(cfg.model.train_ds)\n", + " model.setup_validation_data(cfg.model.validation_ds)\n", + " model.setup_optimization(cfg.model.optim)\n", + " print(\"Warm-starting from \" + warmstart_path)\n", + " exp_manager(trainer, cfg.get('exp_manager', None))\n", + " trainer.callbacks.extend([pl.callbacks.LearningRateMonitor(), LogEpochTimeCallback()]) # noqa\n", + " trainer.fit(model)\n", + "\n", + "os.chdir(cwd)\n", + "GlobalHydra().clear()\n", + "initialize(config_path=\"conf\")\n", + "cfg = compose(config_name=\"talknet-pitch\")\n", + "train(cfg)" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "JLfm00NuJfon" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 6:** Train spectrogram generator. 200+ epochs are recommended. \n", + "\n", + "This is the slowest of the three models to train, and the hardest to\n", + "get good results from. If your character sounds noisy or robotic,\n", + "try improving the dataset, or adjusting the epochs and learning rate.\n", + "\n", + "If CUDA runs out of memory, try the following:\n", + "* Click on Kernel -> Restart, re-run step 1, and try again.\n", + "* If that doesn't help, reduce the batch size (default 32)." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "epochs = 200\n", + "batch_size = 32\n", + "\n", + "# Advanced settings. You can probably leave these at their defaults (1e-3, 3e-6, empty, checked).\n", + "learning_rate = 1e-3\n", + "min_learning_rate = 3e-6\n", + "pretrained_path = \"\"\n", + "load_checkpoints = True\n", + "\n", + "import os\n", + "from hydra.experimental import compose, initialize\n", + "from hydra.core.global_hydra import GlobalHydra\n", + "from omegaconf import OmegaConf\n", + "import pytorch_lightning as pl\n", + "from nemo.collections.common.callbacks import LogEpochTimeCallback\n", + "from nemo.collections.tts.models import TalkNetSpectModel\n", + "from nemo.core.config import hydra_runner\n", + "from nemo.utils.exp_manager import exp_manager\n", + "\n", + "def train(cfg):\n", + " cfg.sample_rate = 22050\n", + " cfg.train_dataset = os.path.join(output_dir, \"trainfiles.json\")\n", + " cfg.validation_datasets = os.path.join(output_dir, \"valfiles.json\")\n", + " cfg.durs_file = os.path.join(output_dir, \"durations.pt\")\n", + " cfg.f0_file = os.path.join(output_dir, \"f0s.pt\")\n", + " cfg.trainer.accelerator = \"dp\"\n", + " cfg.trainer.max_epochs = epochs\n", + " cfg.trainer.check_val_every_n_epoch = 5\n", + " cfg.model.train_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.validation_ds.dataloader_params.batch_size = batch_size\n", + " cfg.model.optim.lr = learning_rate\n", + " cfg.model.optim.sched.min_lr = min_learning_rate\n", + " cfg.exp_manager.exp_dir = output_dir\n", + "\n", + " # Find checkpoints\n", + " ckpt_path = \"\"\n", + " if load_checkpoints:\n", + " path0 = os.path.join(output_dir, \"TalkNetSpect\")\n", + " if os.path.exists(path0):\n", + " path1 = sorted(os.listdir(path0))\n", + " for i in range(len(path1)):\n", + " path2 = os.path.join(path0, path1[-(1+i)], \"checkpoints\")\n", + " if os.path.exists(path2):\n", + " match = [x for x in os.listdir(path2) if \"last.ckpt\" in x]\n", + " if len(match) > 0:\n", + " ckpt_path = os.path.join(path2, match[0])\n", + " print(\"Resuming training from \" + match[0])\n", + " break\n", + " \n", + " if ckpt_path != \"\":\n", + " trainer = pl.Trainer(**cfg.trainer, resume_from_checkpoint = ckpt_path)\n", + " model = TalkNetSpectModel(cfg=cfg.model, trainer=trainer)\n", + " else:\n", + " if pretrained_path != \"\":\n", + " warmstart_path = pretrained_path\n", + " else:\n", + " warmstart_path = os.path.join(cwd, \"talknet_spect.nemo\")\n", + " trainer = pl.Trainer(**cfg.trainer)\n", + " model = TalkNetSpectModel.restore_from(warmstart_path, override_config_path=cfg)\n", + " model.set_trainer(trainer)\n", + " model.setup_training_data(cfg.model.train_ds)\n", + " model.setup_validation_data(cfg.model.validation_ds)\n", + " model.setup_optimization(cfg.model.optim)\n", + " print(\"Warm-starting from \" + warmstart_path)\n", + " exp_manager(trainer, cfg.get('exp_manager', None))\n", + " trainer.callbacks.extend([pl.callbacks.LearningRateMonitor(), LogEpochTimeCallback()]) # noqa\n", + " trainer.fit(model)\n", + "\n", + "os.chdir(cwd)\n", + "GlobalHydra().clear()\n", + "initialize(config_path=\"conf\")\n", + "cfg = compose(config_name=\"talknet-spect\")\n", + "train(cfg)" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "N9hh4WPHbCcn" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 7:** Generate GTA spectrograms. This will help HiFi-GAN learn what your TalkNet model sounds like.\n", + "\n", + "If this step fails, make sure you've finished training the spectrogram generator." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "import sys\n", + "import os\n", + "import torch\n", + "import numpy as np\n", + "from tqdm import tqdm\n", + "from nemo.collections.tts.models import TalkNetSpectModel\n", + "import shutil\n", + "\n", + "def fix_paths(inpath):\n", + " output = \"\"\n", + " with open(inpath, \"r\", encoding=\"utf8\") as f:\n", + " for l in f.readlines():\n", + " if l[:5].lower() != \"wavs/\":\n", + " output += \"wavs/\" + l\n", + " else:\n", + " output += l\n", + " with open(inpath, \"w\", encoding=\"utf8\") as w:\n", + " w.write(output)\n", + "\n", + "shutil.copyfile(train_filelist, os.path.join(cwd, \"hifi-gan\", \"training.txt\"))\n", + "shutil.copyfile(val_filelist, os.path.join(cwd, \"hifi-gan\", \"validation.txt\"))\n", + "fix_paths(os.path.join(cwd, \"hifi-gan\", \"training.txt\"))\n", + "fix_paths(os.path.join(cwd, \"hifi-gan\", \"validation.txt\"))\n", + "fix_paths(os.path.join(output_dir, \"allfiles.txt\"))\n", + "\n", + "os.chdir(cwd)\n", + "outdir = os.path.join(output_dir, \"_training_data\", \"wavs\")\n", + "if not os.path.exists(outdir):\n", + " os.mkdir(outdir)\n", + "\n", + "model_path = \"\"\n", + "path0 = os.path.join(output_dir, \"TalkNetSpect\")\n", + "if os.path.exists(path0):\n", + " path1 = sorted(os.listdir(path0))\n", + " for i in range(len(path1)):\n", + " path2 = os.path.join(path0, path1[-(1+i)], \"checkpoints\")\n", + " if os.path.exists(path2):\n", + " match = [x for x in os.listdir(path2) if \"TalkNetSpect.nemo\" in x]\n", + " if len(match) > 0:\n", + " model_path = os.path.join(path2, match[0])\n", + " break\n", + "assert model_path != \"\", \"TalkNetSpect.nemo not found\"\n", + "\n", + "dur_path = os.path.join(output_dir, \"durations.pt\")\n", + "f0_path = os.path.join(output_dir, \"f0s.pt\")\n", + "\n", + "model = TalkNetSpectModel.restore_from(model_path)\n", + "model.eval()\n", + "with open(os.path.join(output_dir, \"allfiles.txt\"), \"r\", encoding=\"utf-8\") as f:\n", + " dataset = f.readlines()\n", + "durs = torch.load(dur_path)\n", + "f0s = torch.load(f0_path)\n", + "\n", + "for x in tqdm(dataset):\n", + " x_name = os.path.splitext(os.path.basename(x.split(\"|\")[0].strip()))[0]\n", + " x_tokens = model.parse(text=x.split(\"|\")[1].strip())\n", + " x_durs = (\n", + " torch.stack(\n", + " (\n", + " durs[x_name][\"blanks\"],\n", + " torch.cat((durs[x_name][\"tokens\"], torch.zeros(1).int())),\n", + " ),\n", + " dim=1,\n", + " )\n", + " .view(-1)[:-1]\n", + " .view(1, -1)\n", + " .to(\"cuda:0\")\n", + " )\n", + " x_f0s = f0s[x_name].view(1, -1).to(\"cuda:0\")\n", + " x_spect = model.force_spectrogram(tokens=x_tokens, durs=x_durs, f0=x_f0s)\n", + " rel_path = os.path.splitext(x.split(\"|\")[0].strip())[0][5:]\n", + " abs_dir = os.path.join(outdir, os.path.dirname(rel_path))\n", + " if abs_dir != \"\" and not os.path.exists(abs_dir):\n", + " os.makedirs(abs_dir, exist_ok=True)\n", + " np.save(os.path.join(outdir, rel_path + \".npy\"), x_spect.detach().cpu().numpy())\n" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "Ajfyfz2p9Ior" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 8:** Train HiFi-GAN. 2,000+ steps are recommended.\n", + "Stop this cell to finish training the model.\n", + "\n", + "* Click on Kernel -> Restart, re-run step 1, and try again.\n", + "If this step still fails to start, make sure step 7 finished successfully.\n", + "\n", + "Note: If the training process starts at step 2500000, delete the HiFiGAN folder and try again." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "import gdown\n", + "d = 'https://drive.google.com/uc?id='\n", + "\n", + "os.chdir(os.path.join(cwd, \"hifi-gan\"))\n", + "\n", + "if not os.path.exists(os.path.join(output_dir, \"HiFiGAN\")):\n", + " os.makedirs(os.path.join(output_dir, \"HiFiGAN\"))\n", + "if not os.path.exists(os.path.join(output_dir, \"HiFiGAN\", \"do_00000000\")):\n", + " print(\"Downloading universal model...\")\n", + " gdown.download(d+\"1qpgI41wNXFcH-iKq1Y42JlBC9j0je8PW\", os.path.join(output_dir, \"HiFiGAN\", \"g_00000000\"), quiet=False)\n", + " gdown.download(d+\"1O63eHZR9t1haCdRHQcEgMfMNxiOciSru\", os.path.join(output_dir, \"HiFiGAN\", \"do_00000000\"), quiet=False)\n", + " start_from_universal = \"--warm_start True \"\n", + "else:\n", + " start_from_universal = \"\"\n", + "\n", + "hifi_train = os.path.join(cwd, \"hifi-gan\", \"training.txt\")\n", + "hifi_val = os.path.join(cwd, \"hifi-gan\", \"validation.txt\")\n", + "hifi_wavs = os.path.join(output_dir, \"_training_data\")\n", + "\n", + "!python train.py --fine_tuning True --config config_v1b.json \\\n", + "{start_from_universal} \\\n", + "--checkpoint_interval 250 --checkpoint_path \"{os.path.join(output_dir, 'HiFiGAN')}\" \\\n", + "--input_training_file \"{hifi_train}\" \\\n", + "--input_validation_file \"{hifi_val}\" \\\n", + "--input_wavs_dir \"{hifi_wavs}\"\n" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "yVBjGhRB9hUJ" + } + }, + { + "cell_type": "markdown", + "source": [ + "**Step 9:** Package the models. They'll be saved to the output directory as [character_name]_TalkNet.zip.\n", + "\n", + "When done, upload it to Google Drive, with permissions set to \"Anyone with the link\". \n", + "You can then use it with TalkNet by selecting \"Custom model\" as your character. \n", + "\n", + "This cell will also delete the training checkpoints and logs.\n", + "That should free up roughly 2 GB of space.\n", + "If you wish to keep them, set delete_checkpoints to False." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "character_name = \"Character\"\n", + "delete_checkpoints = True\n", + "\n", + "import shutil\n", + "from zipfile import ZipFile\n", + "\n", + "def find_talknet(model_dir):\n", + " ckpt_path = \"\"\n", + " path0 = os.path.join(output_dir, model_dir)\n", + " if os.path.exists(path0):\n", + " path1 = sorted(os.listdir(path0))\n", + " for i in range(len(path1)):\n", + " path2 = os.path.join(path0, path1[-(1+i)], \"checkpoints\")\n", + " if os.path.exists(path2):\n", + " match = [x for x in os.listdir(path2) if \".nemo\" in x]\n", + " if len(match) > 0:\n", + " ckpt_path = os.path.join(path2, match[0])\n", + " break\n", + " assert ckpt_path != \"\", \"Couldn't find \" + model_dir\n", + " return ckpt_path\n", + "\n", + "durs_path = find_talknet(\"TalkNetDurs\")\n", + "pitch_path = find_talknet(\"TalkNetPitch\")\n", + "spect_path = find_talknet(\"TalkNetSpect\")\n", + "assert os.path.exists(os.path.join(output_dir, \"HiFiGAN\", \"g_00000000\")), \"Couldn't find HiFi-GAN\"\n", + "\n", + "zip = ZipFile(os.path.join(output_dir, character_name + \"_TalkNet.zip\"), 'w')\n", + "zip.write(durs_path, \"TalkNetDurs.nemo\")\n", + "zip.write(pitch_path, \"TalkNetPitch.nemo\")\n", + "zip.write(spect_path, \"TalkNetSpect.nemo\")\n", + "zip.write(os.path.join(output_dir, \"HiFiGAN\", \"g_00000000\"), \"hifiganmodel\")\n", + "zip.write(os.path.join(output_dir, \"HiFiGAN\", \"config.json\"), \"config.json\")\n", + "zip.write(os.path.join(output_dir, \"f0_info.json\"), \"f0_info.json\")\n", + "zip.close()\n", + "print(\"Archived model to \" + os.path.join(output_dir, character_name + \"_TalkNet.zip\"))\n", + "\n", + "if delete_checkpoints:\n", + " shutil.rmtree((os.path.join(output_dir, \"TalkNetDurs\")))\n", + " shutil.rmtree((os.path.join(output_dir, \"TalkNetPitch\")))\n", + " shutil.rmtree((os.path.join(output_dir, \"TalkNetSpect\")))\n", + " shutil.rmtree((os.path.join(output_dir, \"HiFiGAN\")))\n" + ], + "outputs": [], + "metadata": { + "cellView": "form", + "id": "5OtwfrTT-blU" + } + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "TalkNet_Training_Offline.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/models/tacotron2/ControllableTalkNet/assets/Equestria.woff b/models/tacotron2/ControllableTalkNet/assets/Equestria.woff new file mode 100644 index 0000000000..6234e41d0c Binary files /dev/null and b/models/tacotron2/ControllableTalkNet/assets/Equestria.woff differ diff --git a/models/tacotron2/ControllableTalkNet/assets/dash_customfont.css b/models/tacotron2/ControllableTalkNet/assets/dash_customfont.css new file mode 100644 index 0000000000..c9855c1568 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/assets/dash_customfont.css @@ -0,0 +1,420 @@ +@font-face { + font-family: 'EquestriaWebfont'; + src: url('Equestria.woff') format('woff'); +} + +/* Table of contents +–––––––––––––––––––––––––––––––––––––––––––––––––– +- Plotly.js +- Grid +- Base Styles +- Typography +- Links +- Buttons +- Forms +- Lists +- Code +- Tables +- Spacing +- Utilities +- Clearing +- Media Queries +*/ + +/* PLotly.js +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +/* plotly.js's modebar's z-index is 1001 by default + * https://github.com/plotly/plotly.js/blob/7e4d8ab164258f6bd48be56589dacd9bdd7fded2/src/css/_modebar.scss#L5 + * In case a dropdown is above the graph, the dropdown's options + * will be rendered below the modebar + * Increase the select option's z-index + */ + +/* This was actually not quite right - + dropdowns were overlapping each other (edited October 26) + +.Select { + z-index: 1002; +}*/ + + +/* Grid +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +.container { + position: relative; + width: 100%; + max-width: 960px; + margin: 0 auto; + padding: 0 20px; + box-sizing: border-box; } +.column, +.columns { + width: 100%; + float: left; + box-sizing: border-box; } + +/* For devices larger than 400px */ +@media (min-width: 400px) { + .container { + width: 85%; + padding: 0; } +} + +/* For devices larger than 550px */ +@media (min-width: 550px) { + .container { + width: 80%; } + .column, + .columns { + margin-left: 4%; } + .column:first-child, + .columns:first-child { + margin-left: 0; } + + .one.column, + .one.columns { width: 4.66666666667%; } + .two.columns { width: 13.3333333333%; } + .three.columns { width: 22%; } + .four.columns { width: 30.6666666667%; } + .five.columns { width: 39.3333333333%; } + .six.columns { width: 48%; } + .seven.columns { width: 56.6666666667%; } + .eight.columns { width: 65.3333333333%; } + .nine.columns { width: 74.0%; } + .ten.columns { width: 82.6666666667%; } + .eleven.columns { width: 91.3333333333%; } + .twelve.columns { width: 100%; margin-left: 0; } + + .one-third.column { width: 30.6666666667%; } + .two-thirds.column { width: 65.3333333333%; } + + .one-half.column { width: 48%; } + + /* Offsets */ + .offset-by-one.column, + .offset-by-one.columns { margin-left: 8.66666666667%; } + .offset-by-two.column, + .offset-by-two.columns { margin-left: 17.3333333333%; } + .offset-by-three.column, + .offset-by-three.columns { margin-left: 26%; } + .offset-by-four.column, + .offset-by-four.columns { margin-left: 34.6666666667%; } + .offset-by-five.column, + .offset-by-five.columns { margin-left: 43.3333333333%; } + .offset-by-six.column, + .offset-by-six.columns { margin-left: 52%; } + .offset-by-seven.column, + .offset-by-seven.columns { margin-left: 60.6666666667%; } + .offset-by-eight.column, + .offset-by-eight.columns { margin-left: 69.3333333333%; } + .offset-by-nine.column, + .offset-by-nine.columns { margin-left: 78.0%; } + .offset-by-ten.column, + .offset-by-ten.columns { margin-left: 86.6666666667%; } + .offset-by-eleven.column, + .offset-by-eleven.columns { margin-left: 95.3333333333%; } + + .offset-by-one-third.column, + .offset-by-one-third.columns { margin-left: 34.6666666667%; } + .offset-by-two-thirds.column, + .offset-by-two-thirds.columns { margin-left: 69.3333333333%; } + + .offset-by-one-half.column, + .offset-by-one-half.columns { margin-left: 52%; } + +} + + +/* Base Styles +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +/* NOTE +html is set to 62.5% so that all the REM measurements throughout Skeleton +are based on 10px sizing. So basically 1.5rem = 15px :) */ +html { + font-size: 62.5%; } +body { + font-size: 1.5em; /* currently ems cause chrome bug misinterpreting rems on body element */ + line-height: 1.6; + font-weight: 400; + font-family: "Open Sans", "HelveticaNeue", "Helvetica Neue", Helvetica, Arial, sans-serif; + color: rgb(50, 50, 50); } + + +/* Typography +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +h1, h2, h3, h4, h5, h6 { + margin-top: 0; + margin-bottom: 0; + font-weight: 300; } +h1 { font-size: 4.5rem; line-height: 1.2; letter-spacing: -.1rem; margin-bottom: 2rem; } +h2 { font-size: 3.6rem; line-height: 1.25; letter-spacing: -.1rem; margin-bottom: 1.8rem; margin-top: 1.8rem;} +h3 { font-size: 3.0rem; line-height: 1.3; letter-spacing: -.1rem; margin-bottom: 1.5rem; margin-top: 1.5rem;} +h4 { font-size: 2.6rem; line-height: 1.35; letter-spacing: -.08rem; margin-bottom: 1.2rem; margin-top: 1.2rem;} +h5 { font-size: 2.2rem; line-height: 1.5; letter-spacing: -.05rem; margin-bottom: 0.6rem; margin-top: 0.6rem;} +h6 { font-size: 2.0rem; line-height: 1.6; letter-spacing: 0; margin-bottom: 0.75rem; margin-top: 0.75rem;} + +p { + margin-top: 0; } + + +/* Blockquotes +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +blockquote { + border-left: 4px lightgrey solid; + padding-left: 1rem; + margin-top: 2rem; + margin-bottom: 2rem; + margin-left: 0rem; +} + + +/* Links +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +a { + color: #1EAEDB; + text-decoration: underline; + cursor: pointer;} +a:hover { + color: #0FA0CE; } + + +/* Buttons +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +.button, +button, +input[type="submit"], +input[type="reset"], +input[type="button"] { + display: inline-block; + height: 38px; + padding: 0 30px; + color: #555; + text-align: center; + font-size: 11px; + font-weight: 600; + line-height: 38px; + letter-spacing: .1rem; + text-transform: uppercase; + text-decoration: none; + white-space: nowrap; + background-color: transparent; + border-radius: 4px; + border: 1px solid #bbb; + cursor: pointer; + box-sizing: border-box; } +.button:hover, +button:hover, +input[type="submit"]:hover, +input[type="reset"]:hover, +input[type="button"]:hover, +.button:focus, +button:focus, +input[type="submit"]:focus, +input[type="reset"]:focus, +input[type="button"]:focus { + color: #333; + border-color: #888; + outline: 0; } +.button.button-primary, +button.button-primary, +input[type="submit"].button-primary, +input[type="reset"].button-primary, +input[type="button"].button-primary { + color: #FFF; + background-color: #33C3F0; + border-color: #33C3F0; } +.button.button-primary:hover, +button.button-primary:hover, +input[type="submit"].button-primary:hover, +input[type="reset"].button-primary:hover, +input[type="button"].button-primary:hover, +.button.button-primary:focus, +button.button-primary:focus, +input[type="submit"].button-primary:focus, +input[type="reset"].button-primary:focus, +input[type="button"].button-primary:focus { + color: #FFF; + background-color: #1EAEDB; + border-color: #1EAEDB; } + + +/* Forms +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +input[type="email"], +input[type="number"], +input[type="search"], +input[type="text"], +input[type="tel"], +input[type="url"], +input[type="password"], +textarea, +select { + height: 38px; + padding: 6px 10px; /* The 6px vertically centers text on FF, ignored by Webkit */ + background-color: #fff; + border: 1px solid #D1D1D1; + border-radius: 4px; + box-shadow: none; + box-sizing: border-box; + font-family: inherit; + font-size: inherit; /*https://stackoverflow.com/questions/6080413/why-doesnt-input-inherit-the-font-from-body*/} +/* Removes awkward default styles on some inputs for iOS */ +input[type="email"], +input[type="number"], +input[type="search"], +input[type="text"], +input[type="tel"], +input[type="url"], +input[type="password"], +textarea { + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; } +textarea { + min-height: 65px; + padding-top: 6px; + padding-bottom: 6px; } +input[type="email"]:focus, +input[type="number"]:focus, +input[type="search"]:focus, +input[type="text"]:focus, +input[type="tel"]:focus, +input[type="url"]:focus, +input[type="password"]:focus, +textarea:focus, +select:focus { + border: 1px solid #33C3F0; + outline: 0; } +label, +legend { + display: block; + margin-bottom: 0px; } +fieldset { + padding: 0; + border-width: 0; } +input[type="checkbox"], +input[type="radio"] { + display: inline; } +label > .label-body { + display: inline-block; + margin-left: .5rem; + font-weight: normal; } + + +/* Lists +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +ul { + list-style: circle inside; } +ol { + list-style: decimal inside; } +ol, ul { + padding-left: 0; + margin-top: 0; } +ul ul, +ul ol, +ol ol, +ol ul { + margin: 1.5rem 0 1.5rem 3rem; + font-size: 90%; } +li { + margin-bottom: 1rem; } + + +/* Tables +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +table { + border-collapse: collapse; +} +th:not(.CalendarDay), +td:not(.CalendarDay) { + padding: 12px 15px; + text-align: left; + border-bottom: 1px solid #E1E1E1; } +th:first-child:not(.CalendarDay), +td:first-child:not(.CalendarDay) { + padding-left: 0; } +th:last-child:not(.CalendarDay), +td:last-child:not(.CalendarDay) { + padding-right: 0; } + + +/* Spacing +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +button, +.button { + margin-bottom: 0rem; } +input, +textarea, +select, +fieldset { + margin-bottom: 0rem; } +pre, +dl, +figure, +table, +form { + margin-bottom: 0rem; } +p, +ul, +ol { + margin-bottom: 0.75rem; } + +/* Utilities +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +.u-full-width { + width: 100%; + box-sizing: border-box; } +.u-max-full-width { + max-width: 100%; + box-sizing: border-box; } +.u-pull-right { + float: right; } +.u-pull-left { + float: left; } + + +/* Misc +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +hr { + margin-top: 3rem; + margin-bottom: 3.5rem; + border-width: 0; + border-top: 1px solid #E1E1E1; } + + +/* Clearing +–––––––––––––––––––––––––––––––––––––––––––––––––– */ + +/* Self Clearing Goodness */ +.container:after, +.row:after, +.u-cf { + content: ""; + display: table; + clear: both; } + + +/* Media Queries +–––––––––––––––––––––––––––––––––––––––––––––––––– */ +/* +Note: The best way to structure the use of media queries is to create the queries +near the relevant code. For example, if you wanted to change the styles for buttons +on small devices, paste the mobile query code up in the buttons section and style it +there. +*/ + + +/* Larger than mobile */ +@media (min-width: 400px) {} + +/* Larger than phablet (also point when grid becomes active) */ +@media (min-width: 550px) {} + +/* Larger than tablet */ +@media (min-width: 750px) {} + +/* Larger than desktop */ +@media (min-width: 1000px) {} + +/* Larger than Desktop HD */ +@media (min-width: 1200px) {} \ No newline at end of file diff --git a/models/tacotron2/ControllableTalkNet/controllable_talknet.py b/models/tacotron2/ControllableTalkNet/controllable_talknet.py new file mode 100644 index 0000000000..3f68cc0077 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/controllable_talknet.py @@ -0,0 +1,1097 @@ +import sys +import os +import base64 +import dash +from jupyter_dash import JupyterDash +import dash_core_components as dcc +import dash_html_components as html +import dash_bootstrap_components as dbc +from dash.exceptions import PreventUpdate +import torch +import numpy as np +import tensorflow as tf +import crepe +import scipy +from scipy.io import wavfile +import psola +import io +import nemo +from nemo.collections.asr.models import EncDecCTCModel +from nemo.collections.tts.models import TalkNetSpectModel +from nemo.collections.tts.models import TalkNetPitchModel +from nemo.collections.tts.models import TalkNetDursModel +from talknet_singer import TalkNetSingerModel +import json +from tqdm import tqdm +import gdown +import zipfile +import resampy +import traceback +import ffmpeg +import time +import uuid + +sys.path.append("hifi-gan") +from env import AttrDict +from meldataset import mel_spectrogram, MAX_WAV_VALUE +from models import Generator +from denoiser import Denoiser + +app = JupyterDash(__name__) +DEVICE = "cuda:0" +CPU_PITCH = False +RUN_PATH = os.path.dirname(os.path.realpath(__file__)) +if RUN_PATH == "/content": + UI_MODE = "colab" +elif os.path.exists("/talknet/is_docker"): + UI_MODE = "docker" +else: + UI_MODE = "offline" +torch.set_grad_enabled(False) +if CPU_PITCH: + tf.config.set_visible_devices([], "GPU") + +app.title = "Controllable TalkNet" +app.layout = html.Div( + children=[ + html.H1( + id="header", + children="Controllable TalkNet", + style={ + "font-family": "EquestriaWebfont", + "color": "#ed3c96", + "font-size": "4em", + "text-align": "center", + "margin-top": "0em", + "margin-bottom": "0em", + }, + ), + html.Label("Character selection", htmlFor="model-dropdown"), + dbc.Select( + id="model-dropdown", + options=[ + { + "label": "Custom model", + "value": "Custom", + }, + { + "label": "--- ERROR LOADING MODEL LISTS ---", + "value": "", + "disabled": True, + }, + ], + value=None, + style={ + "max-width": "90vw", + "width": "35em", + "margin-bottom": "0.7em", + }, + ), + html.Div( + children=[ + dcc.Input( + id="drive-id", + type="text", + placeholder="Drive ID for custom model", + style={"width": "22em"}, + ), + ], + id="custom-model", + style={ + "display": "none", + }, + ), + html.Label( + "Upload reference audio to " + RUN_PATH, + htmlFor="reference-dropdown", + id="upload-label", + ), + dcc.Upload( + id="upload-audio", + children=html.Div(["Drag and drop or click to select a file to upload."]), + style={ + "display": "none", + }, + multiple=True, + ), + dcc.Store(id="current-f0s"), + dcc.Store(id="current-f0s-nosilence"), + dcc.Store(id="current-filename"), + dcc.Loading( + id="audio-loading", + children=[ + html.Div( + [ + html.Button( + "Update file list", + id="update-button", + style={ + "margin-right": "10px", + }, + ), + dbc.Select( + id="reference-dropdown", + options=[], + value=None, + style={ + "max-width": "80vw", + "width": "30em", + }, + disabled=False, + ), + dcc.Store(id="pitch-clicks"), + html.Button( + "Debug pitch", + id="pitch-button", + style={ + "margin-left": "10px", + }, + disabled=False, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "row", + "margin-left": "50px", + "vertical-align": "middle", + }, + ), + html.Audio( + id="pitch-out", + controls=True, + style={"display": "none"}, + ), + html.Div( + id="audio-loading-output", + style={ + "font-style": "italic", + "margin-bottom": "0.7em", + "text-align": "center", + }, + ), + ], + type="default", + ), + html.Div( + [ + dcc.Checklist( + id="pitch-options", + options=[ + {"label": "Change input pitch", "value": "pf"}, + {"label": "Auto-tune output", "value": "pc"}, + {"label": "Disable reference audio", "value": "dra"}, + ], + value=[], + ), + html.Div( + [ + html.Label("Semitones", htmlFor="pitch-factor"), + dcc.Input( + id="pitch-factor", + type="number", + value="0", + style={"width": "7em"}, + min=-11, + max=11, + step=1, + disabled=True, + ), + ], + style={ + "flex-direction": "column", + "margin-left": "10px", + "margin-bottom": "0.7em", + }, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "row", + "margin-left": "50px", + "margin-bottom": "0.7em", + }, + ), + html.Label("Transcript", htmlFor="transcript-input"), + dcc.Textarea( + id="transcript-input", + value="", + style={ + "max-width": "90vw", + "width": "50em", + "height": "8em", + "margin-bottom": "0.7em", + }, + ), + dcc.Loading( + html.Div( + [ + html.Button( + "Generate", + id="gen-button", + ), + html.Audio( + id="audio-out", + controls=True, + style={ + "display": "none", + }, + ), + html.Div( + id="generated-info", + style={ + "font-style": "italic", + }, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "column", + }, + ) + ), + html.Footer( + children=""" + Presented by the Pony Preservation Project. + """, + style={"margin-top": "2em", "font-size": "0.7em"}, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "column", + "background-color": "#FFF", + }, +) + +upload_display = { + "width": "100%", + "height": "60px", + "lineHeight": "60px", + "borderWidth": "1px", + "borderStyle": "dashed", + "borderRadius": "5px", + "textAlign": "center", + "margin": "10px", +} + +playback_style = { + "margin-top": "0.3em", + "margin-bottom": "0.3em", + "display": "block", + "width": "600px", + "max-width": "90vw", +} + +playback_hide = { + "display": "none", +} + + +@app.callback( + [ + dash.dependencies.Output("model-dropdown", "options"), + dash.dependencies.Output("upload-audio", "style"), + ], + dash.dependencies.Input("header", "children"), +) +def init_dropdown(value): + if UI_MODE == "docker": + upload_style = upload_display + else: + upload_style = playback_hide + + dropdown = [ + { + "label": "Custom model", + "value": "Custom|default", + } + ] + prev_values = ["Custom|default"] + + def add_to_dropdown(entry): + if entry["value"] in prev_values: + return + dropdown.append(entry) + prev_values.append(entry["value"]) + + all_dict = {} + for filename in os.listdir("model_lists"): + if len(filename) < 5 or filename[-5:].lower() != ".json": + continue + with open(os.path.join("model_lists", filename)) as f: + j = json.load(f) + for s in j: + for c in s["characters"]: + c["source_file"] = filename[:-5] + if s["source"] not in all_dict: + all_dict[s["source"]] = s["characters"] + else: + all_dict[s["source"]].extend(s["characters"]) + for k in sorted(all_dict): + seen_chars = [] + seen_ids = [] + characters = {} + characters_sing = {} + has_singers = False + for c in all_dict[k]: + if c["drive_id"] in seen_ids: + continue + seen_ids.append(c["drive_id"]) + # Handle duplicate names + if c["name"] in seen_chars: + if c["name"] in characters: + rename = ( + c["name"] + " [" + characters[c["name"]]["source_file"] + "]" + ) + characters[rename] = characters[c["name"]] + del characters[c["name"]] + c["name"] = c["name"] + " [" + c["source_file"] + "]" + else: + seen_chars.append(c["name"]) + + characters[c["name"]] = { + "drive_id": c["drive_id"], + "is_singing": c["is_singing"], + "source_file": c["source_file"], + } + if c["is_singing"]: + has_singers = True + if has_singers: + for ck in sorted(characters): + if characters[ck]["is_singing"]: + characters_sing[ck] = characters[ck] + del characters[ck] + separator = "--- " + k.strip().upper() + " MODELS (TALKING) ---" + else: + separator = "--- " + k.strip().upper() + " MODELS ---" + if len(characters) > 0: + add_to_dropdown( + { + "label": separator, + "value": str(uuid.uuid4()) + "|default", + "disabled": True, + } + ) + for ck in sorted(characters): + add_to_dropdown( + { + "label": ck, + "value": characters[ck]["drive_id"] + "|default", + } + ) + if has_singers: + separator = "--- " + k.strip().upper() + " MODELS (SINGING) ---" + add_to_dropdown( + { + "label": separator, + "value": str(uuid.uuid4()) + "|default", + "disabled": True, + } + ) + for ck in sorted(characters_sing): + add_to_dropdown( + { + "label": ck, + "value": characters_sing[ck]["drive_id"] + "|singing", + } + ) + if len(all_dict) == 0: + add_to_dropdown( + { + "label": "--- NO MODEL LISTS FOUND ---", + "value": str(uuid.uuid4()) + "|default", + "disabled": True, + } + ) + return [dropdown, upload_style] + + +def load_hifigan(model_name, conf_name): + # Load HiFi-GAN + conf = os.path.join("hifi-gan", conf_name + ".json") + with open(conf) as f: + json_config = json.loads(f.read()) + h = AttrDict(json_config) + torch.manual_seed(h.seed) + hifigan = Generator(h).to(torch.device(DEVICE)) + state_dict_g = torch.load(model_name, map_location=torch.device(DEVICE)) + hifigan.load_state_dict(state_dict_g["generator"]) + hifigan.eval() + hifigan.remove_weight_norm() + denoiser = Denoiser(hifigan, mode="normal") + return hifigan, h, denoiser + + +def generate_json(input, outpath): + output = "" + sample_rate = 22050 + lpath = input.split("|")[0].strip() + size = os.stat(lpath).st_size + x = { + "audio_filepath": lpath, + "duration": size / (sample_rate * 2), + "text": input.split("|")[1].strip(), + } + output += json.dumps(x) + "\n" + with open(outpath, "w", encoding="utf8") as w: + w.write(output) + + +asr_model = ( + EncDecCTCModel.from_pretrained(model_name="asr_talknet_aligner").cpu().eval() +) + + +def forward_extractor(tokens, log_probs, blank): + """Computes states f and p.""" + n, m = len(tokens), log_probs.shape[0] + # `f[s, t]` -- max sum of log probs for `s` first codes + # with `t` first timesteps with ending in `tokens[s]`. + f = np.empty((n + 1, m + 1), dtype=float) + f.fill(-(10 ** 9)) + p = np.empty((n + 1, m + 1), dtype=int) + f[0, 0] = 0.0 # Start + for s in range(1, n + 1): + c = tokens[s - 1] + for t in range((s + 1) // 2, m + 1): + f[s, t] = log_probs[t - 1, c] + # Option #1: prev char is equal to current one. + if s == 1 or c == blank or c == tokens[s - 3]: + options = f[s : (s - 2 if s > 1 else None) : -1, t - 1] + else: # Is not equal to current one. + options = f[s : (s - 3 if s > 2 else None) : -1, t - 1] + f[s, t] += np.max(options) + p[s, t] = np.argmax(options) + return f, p + + +def backward_extractor(f, p): + """Computes durs from f and p.""" + n, m = f.shape + n -= 1 + m -= 1 + durs = np.zeros(n, dtype=int) + if f[-1, -1] >= f[-2, -1]: + s, t = n, m + else: + s, t = n - 1, m + while s > 0: + durs[s - 1] += 1 + s -= p[s, t] + t -= 1 + assert durs.shape[0] == n + assert np.sum(durs) == m + assert np.all(durs[1::2] > 0) + return durs + + +def preprocess_tokens(tokens, blank): + new_tokens = [blank] + for c in tokens: + new_tokens.extend([c, blank]) + tokens = new_tokens + return tokens + + +parser = ( + nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset.make_vocab( + notation="phonemes", + punct=True, + spaces=True, + stresses=False, + add_blank_at="last", + ) +) + + +def arpa_parse(input, model): + z = [] + space = parser.labels.index(" ") + while "{" in input: + if "}" not in input: + input.replace("{", "") + else: + pre = input[: input.find("{")] + if pre.strip() != "": + x = model.parse(text=pre.strip()) + seq_ids = x.squeeze(0).cpu().detach().numpy() + z.extend(seq_ids) + z.append(space) + + arpaword = input[input.find("{") + 1 : input.find("}")] + arpaword = ( + arpaword.replace("0", "") + .replace("1", "") + .replace("2", "") + .strip() + .split(" ") + ) + + seq_ids = [] + for x in arpaword: + if x == "": + continue + if x.replace("_", " ") not in parser.labels: + continue + seq_ids.append(parser.labels.index(x.replace("_", " "))) + seq_ids.append(space) + z.extend(seq_ids) + input = input[input.find("}") + 1 :] + if input != "": + x = model.parse(text=input.strip()) + seq_ids = x.squeeze(0).cpu().detach().numpy() + z.extend(seq_ids) + if z[-1] == space: + z = z[:-1] + if z[0] == space: + z = z[1:] + return [ + z[i] for i in range(len(z)) if (i == 0) or (z[i] != z[i - 1]) or (z[i] != space) + ] + + +def to_arpa(input): + arpa = "" + z = [] + space = parser.labels.index(" ") + while space in input: + z.append(input[: input.index(space)]) + input = input[input.index(space) + 1 :] + z.append(input) + for y in z: + if len(y) == 0: + continue + + arpaword = " {" + for s in y: + if parser.labels[s] == " ": + arpaword += "_ " + else: + arpaword += parser.labels[s] + " " + arpaword += "} " + if not arpaword.replace("{", "").replace("}", "").replace(" ", "").isalnum(): + arpaword = arpaword.replace("{", "").replace(" }", "") + arpa += arpaword + return arpa.replace(" ", " ").replace(" }", "}").strip() + + +def get_duration(wav_name, transcript, tokens): + if not os.path.exists(os.path.join(RUN_PATH, "temp")): + os.mkdir(os.path.join(RUN_PATH, "temp")) + if "_" not in transcript: + generate_json( + os.path.join(RUN_PATH, "temp", wav_name + "_conv.wav") + + "|" + + transcript.strip(), + os.path.join(RUN_PATH, "temp", wav_name + ".json"), + ) + else: + generate_json( + os.path.join(RUN_PATH, "temp", wav_name + "_conv.wav") + "|" + "dummy", + os.path.join(RUN_PATH, "temp", wav_name + ".json"), + ) + + data_config = { + "manifest_filepath": os.path.join(RUN_PATH, "temp", wav_name + ".json"), + "sample_rate": 22050, + "batch_size": 1, + } + + dataset = nemo.collections.asr.data.audio_to_text._AudioTextDataset( + manifest_filepath=data_config["manifest_filepath"], + sample_rate=data_config["sample_rate"], + parser=parser, + ) + + dl = torch.utils.data.DataLoader( + dataset=dataset, + batch_size=data_config["batch_size"], + collate_fn=dataset.collate_fn, + shuffle=False, + ) + + blank_id = asr_model.decoder.num_classes_with_blank - 1 + + for sample_idx, test_sample in tqdm(enumerate(dl), total=len(dl)): + log_probs, _, greedy_predictions = asr_model( + input_signal=test_sample[0], input_signal_length=test_sample[1] + ) + + log_probs = log_probs[0].cpu().detach().numpy() + target_tokens = preprocess_tokens(tokens, blank_id) + + f, p = forward_extractor(target_tokens, log_probs, blank_id) + durs = backward_extractor(f, p) + + del test_sample + return durs + return None + + +def crepe_f0(wav_path, hop_length=256): + # sr, audio = wavfile.read(io.BytesIO(wav_data)) + sr, audio = wavfile.read(wav_path) + audio_x = np.arange(0, len(audio)) / 22050.0 + f0time, frequency, confidence, activation = crepe.predict(audio, sr, viterbi=True) + + x = np.arange(0, len(audio), hop_length) / 22050.0 + freq_interp = np.interp(x, f0time, frequency) + conf_interp = np.interp(x, f0time, confidence) + audio_interp = np.interp(x, audio_x, np.absolute(audio)) / 32768.0 + weights = [0.5, 0.25, 0.25] + audio_smooth = np.convolve(audio_interp, np.array(weights)[::-1], "same") + + conf_threshold = 0.25 + audio_threshold = 0.0005 + for i in range(len(freq_interp)): + if conf_interp[i] < conf_threshold: + freq_interp[i] = 0.0 + if audio_smooth[i] < audio_threshold: + freq_interp[i] = 0.0 + + # Hack to make f0 and mel lengths equal + if len(audio) % hop_length == 0: + freq_interp = np.pad(freq_interp, pad_width=[0, 1]) + return ( + torch.from_numpy(freq_interp.astype(np.float32)), + torch.from_numpy(frequency.astype(np.float32)), + ) + + +def f0_to_audio(f0s): + volume = 0.2 + sr = 22050 + freq = 440.0 + base_audio = ( + np.sin(2 * np.pi * np.arange(256.0 * len(f0s)) * freq / sr) * volume + ).astype(np.float32) + shifted_audio = psola.vocode(base_audio, sr, target_pitch=f0s) + for i in range(len(f0s)): + if f0s[i] == 0.0: + shifted_audio[i * 256 : (i + 1) * 256] = 0.0 + print(type(shifted_audio[0])) + buffer = io.BytesIO() + wavfile.write(buffer, sr, shifted_audio.astype(np.float32)) + b64 = base64.b64encode(buffer.getvalue()) + sound = "data:audio/x-wav;base64," + b64.decode("ascii") + return sound + + +@app.callback( + dash.dependencies.Output("custom-model", "style"), + dash.dependencies.Input("model-dropdown", "value"), +) +def update_model(model): + if model is not None and model.split("|")[0] == "Custom": + style = {"margin-bottom": "0.7em", "display": "block"} + else: + style = {"display": "none"} + return style + + +@app.callback( + [ + dash.dependencies.Output("pitch-factor", "disabled"), + dash.dependencies.Output("reference-dropdown", "disabled"), + dash.dependencies.Output("pitch-button", "disabled"), + ], + [ + dash.dependencies.Input("pitch-options", "value"), + ], +) +def update_pitch_options(value): + return ["pf" not in value, "dra" in value, "dra" in value] + + +@app.callback( + dash.dependencies.Output("upload-label", "children"), + [ + dash.dependencies.Input("upload-audio", "filename"), + dash.dependencies.Input("upload-audio", "contents"), + ], +) +def save_upload(uploaded_filenames, uploaded_file_contents): + try: + if uploaded_filenames is not None and uploaded_file_contents is not None: + for name, content in zip(uploaded_filenames, uploaded_file_contents): + if name.strip() == "": + continue + data = content.encode("utf8").split(b";base64,")[1] + with open(os.path.join(RUN_PATH, name), "wb") as fp: + fp.write(base64.decodebytes(data)) + except Exception as e: + return str(e) + return "Uploaded " + str(len(uploaded_filenames)) + " file(s)" + + +@app.callback( + dash.dependencies.Output("reference-dropdown", "options"), + [ + dash.dependencies.Input("update-button", "n_clicks"), + ], +) +def update_filelist(n_clicks): + filelist = [] + supported_formats = [".wav", ".ogg", ".mp3", "flac", ".aac"] + for x in sorted(os.listdir(RUN_PATH)): + if x[-4:].lower() in supported_formats: + filelist.append({"label": x, "value": x}) + return filelist + + +@app.callback( + [ + dash.dependencies.Output("audio-loading-output", "children"), + dash.dependencies.Output("current-f0s", "data"), + dash.dependencies.Output("current-f0s-nosilence", "data"), + dash.dependencies.Output("current-filename", "data"), + ], + [ + dash.dependencies.Input("reference-dropdown", "value"), + ], +) +def select_file(dropdown_value): + if dropdown_value is not None: + if not os.path.exists(os.path.join(RUN_PATH, "temp")): + os.mkdir(os.path.join(RUN_PATH, "temp")) + ffmpeg.input(os.path.join(RUN_PATH, dropdown_value)).output( + os.path.join(RUN_PATH, "temp", dropdown_value + "_conv.wav"), + ar="22050", + ac="1", + acodec="pcm_s16le", + map_metadata="-1", + fflags="+bitexact", + ).overwrite_output().run(quiet=True) + fo_with_silence, f0_wo_silence = crepe_f0( + os.path.join(RUN_PATH, "temp", dropdown_value + "_conv.wav") + ) + return [ + "Analyzed " + dropdown_value, + fo_with_silence, + f0_wo_silence, + dropdown_value, + ] + else: + return ["No audio analyzed", None, None] + + +@app.callback( + [ + dash.dependencies.Output("pitch-out", "src"), + dash.dependencies.Output("pitch-out", "style"), + dash.dependencies.Output("pitch-clicks", "data"), + ], + [ + dash.dependencies.Input("pitch-button", "n_clicks"), + dash.dependencies.Input("pitch-clicks", "data"), + dash.dependencies.Input("current-f0s", "data"), + ], +) +def debug_pitch(n_clicks, pitch_clicks, current_f0s): + if not n_clicks or current_f0s is None or n_clicks <= pitch_clicks: + if n_clicks is not None: + pitch_clicks = n_clicks + else: + pitch_clicks = 0 + return [ + None, + playback_hide, + pitch_clicks, + ] + pitch_clicks = n_clicks + return [f0_to_audio(current_f0s), playback_style, pitch_clicks] + + +hifigan_sr = None + + +def download_model(model, custom_model): + try: + global hifigan_sr, h2, denoiser_sr + d = "https://drive.google.com/uc?id=" + if model == "Custom": + drive_id = custom_model + else: + drive_id = model + if drive_id == "" or drive_id is None: + return ("Missing Drive ID", None, None) + if not os.path.exists(os.path.join(RUN_PATH, "models")): + os.mkdir(os.path.join(RUN_PATH, "models")) + if not os.path.exists(os.path.join(RUN_PATH, "models", drive_id)): + os.mkdir(os.path.join(RUN_PATH, "models", drive_id)) + zip_path = os.path.join(RUN_PATH, "models", drive_id, "model.zip") + gdown.download( + d + drive_id, + zip_path, + quiet=False, + ) + if not os.path.exists(zip_path): + os.rmdir(os.path.join(RUN_PATH, "models", drive_id)) + return ("Model download failed", None, None) + if os.stat(zip_path).st_size < 16: + os.remove(zip_path) + os.rmdir(os.path.join(RUN_PATH, "models", drive_id)) + return ("Model zip is empty", None, None) + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(os.path.join(RUN_PATH, "models", drive_id)) + os.remove(zip_path) + + # Download super-resolution HiFi-GAN + sr_path = "hifi-gan/hifisr" + if not os.path.exists(sr_path): + gdown.download( + d + "14fOprFAIlCQkVRxsfInhEPG0n-xN4QOa", sr_path, quiet=False + ) + if not os.path.exists(sr_path): + raise Exception("HiFI-GAN model failed to download!") + if hifigan_sr is None: + hifigan_sr, h2, denoiser_sr = load_hifigan(sr_path, "config_32k") + + return ( + None, + os.path.join(RUN_PATH, "models", drive_id, "TalkNetSpect.nemo"), + os.path.join(RUN_PATH, "models", drive_id, "hifiganmodel"), + ) + except Exception as e: + return (str(e), None, None) + + +tnmodel, tnpath, tndurs, tnpitch = None, None, None, None +hifigan, h, denoiser, hifipath = None, None, None, None + + +@app.callback( + [ + dash.dependencies.Output("audio-out", "src"), + dash.dependencies.Output("generated-info", "children"), + dash.dependencies.Output("audio-out", "style"), + dash.dependencies.Output("audio-out", "title"), + ], + [dash.dependencies.Input("gen-button", "n_clicks")], + [ + dash.dependencies.State("model-dropdown", "value"), + dash.dependencies.State("drive-id", "value"), + dash.dependencies.State("transcript-input", "value"), + dash.dependencies.State("pitch-options", "value"), + dash.dependencies.State("pitch-factor", "value"), + dash.dependencies.State("current-filename", "data"), + dash.dependencies.State("current-f0s", "data"), + dash.dependencies.State("current-f0s-nosilence", "data"), + ], +) +def generate_audio( + n_clicks, + model, + custom_model, + transcript, + pitch_options, + pitch_factor, + wav_name, + f0s, + f0s_wo_silence, +): + global tnmodel, tnpath, tndurs, tnpitch, hifigan, h, denoiser, hifipath + + if n_clicks is None: + raise PreventUpdate + if model is None: + return [None, "No character selected", playback_hide, None] + if transcript is None or transcript.strip() == "": + return [ + None, + "No transcript entered", + playback_hide, + None, + ] + if wav_name is None and "dra" not in pitch_options: + return [ + None, + "No reference audio selected", + playback_hide, + None, + ] + load_error, talknet_path, hifigan_path = download_model( + model.split("|")[0], custom_model + ) + if load_error is not None: + return [ + None, + load_error, + playback_hide, + None, + ] + + try: + with torch.no_grad(): + if tnpath != talknet_path: + singer_path = os.path.join( + os.path.dirname(talknet_path), "TalkNetSinger.nemo" + ) + if os.path.exists(singer_path): + tnmodel = TalkNetSingerModel.restore_from(singer_path) + else: + tnmodel = TalkNetSpectModel.restore_from(talknet_path) + durs_path = os.path.join( + os.path.dirname(talknet_path), "TalkNetDurs.nemo" + ) + pitch_path = os.path.join( + os.path.dirname(talknet_path), "TalkNetPitch.nemo" + ) + if os.path.exists(durs_path): + tndurs = TalkNetDursModel.restore_from(durs_path) + tnmodel.add_module("_durs_model", tndurs) + tnpitch = TalkNetPitchModel.restore_from(pitch_path) + tnmodel.add_module("_pitch_model", tnpitch) + else: + tndurs = None + tnpitch = None + tnmodel.eval() + tnpath = talknet_path + + token_list = arpa_parse(transcript, tnmodel) + tokens = torch.IntTensor(token_list).view(1, -1).to(DEVICE) + arpa = to_arpa(token_list) + + if "dra" in pitch_options: + if tndurs is None or tnpitch is None: + return [ + None, + "Model doesn't support pitch prediction", + playback_hide, + None, + ] + spect = tnmodel.generate_spectrogram(tokens=tokens) + else: + durs = get_duration(wav_name, transcript, token_list) + + # Change pitch + if "pf" in pitch_options: + f0_factor = np.power(np.e, (0.0577623 * float(pitch_factor))) + f0s = [x * f0_factor for x in f0s] + f0s_wo_silence = [x * f0_factor for x in f0s_wo_silence] + + spect = tnmodel.force_spectrogram( + tokens=tokens, + durs=torch.from_numpy(durs) + .view(1, -1) + .type(torch.LongTensor) + .to(DEVICE), + f0=torch.FloatTensor(f0s).view(1, -1).to(DEVICE), + ) + + if hifipath != hifigan_path: + hifigan, h, denoiser = load_hifigan(hifigan_path, "config_v1") + hifipath = hifigan_path + + y_g_hat = hifigan(spect.float()) + audio = y_g_hat.squeeze() + audio = audio * MAX_WAV_VALUE + audio_denoised = denoiser(audio.view(1, -1), strength=35)[:, 0] + audio_np = ( + audio_denoised.detach().cpu().numpy().reshape(-1).astype(np.int16) + ) + + # Auto-tuning + if "pc" in pitch_options and "dra" not in pitch_options: + _, output_freq, _, _ = crepe.predict(audio_np, 22050, viterbi=True) + output_pitch = torch.from_numpy(output_freq.astype(np.float32)) + target_pitch = torch.FloatTensor(f0s_wo_silence) + factor = torch.mean(output_pitch) / torch.mean(target_pitch) + + octaves = [0.125, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0] + nearest_octave = min(octaves, key=lambda x: abs(x - factor)) + target_pitch *= nearest_octave + if len(target_pitch) < len(output_pitch): + target_pitch = torch.nn.functional.pad( + target_pitch, + (0, list(output_pitch.shape)[0] - list(target_pitch.shape)[0]), + "constant", + 0, + ) + if len(target_pitch) > len(output_pitch): + target_pitch = target_pitch[0 : list(output_pitch.shape)[0]] + + audio_np = psola.vocode( + audio_np, 22050, target_pitch=target_pitch + ).astype(np.float32) + normalize = (1.0 / np.max(np.abs(audio_np))) ** 0.9 + audio_np = audio_np * normalize * MAX_WAV_VALUE + audio_np = audio_np.astype(np.int16) + + # Resample to 32k + wave = resampy.resample( + audio_np, + h.sampling_rate, + h2.sampling_rate, + filter="sinc_window", + window=scipy.signal.windows.hann, + num_zeros=8, + ) + wave_out = wave.astype(np.int16) + + # HiFi-GAN super-resolution + wave = wave / MAX_WAV_VALUE + wave = torch.FloatTensor(wave).to(torch.device(DEVICE)) + new_mel = mel_spectrogram( + wave.unsqueeze(0), + h2.n_fft, + h2.num_mels, + h2.sampling_rate, + h2.hop_size, + h2.win_size, + h2.fmin, + h2.fmax, + ) + y_g_hat2 = hifigan_sr(new_mel) + audio2 = y_g_hat2.squeeze() + audio2 = audio2 * MAX_WAV_VALUE + audio2_denoised = denoiser(audio2.view(1, -1), strength=35)[:, 0] + + # High-pass filter, mixing and denormalizing + audio2_denoised = audio2_denoised.detach().cpu().numpy().reshape(-1) + b = scipy.signal.firwin( + 101, cutoff=10500, fs=h2.sampling_rate, pass_zero=False + ) + y = scipy.signal.lfilter(b, [1.0], audio2_denoised) + y *= 4.0 # superres strength + y_out = y.astype(np.int16) + y_padded = np.zeros(wave_out.shape) + y_padded[: y_out.shape[0]] = y_out + sr_mix = wave_out + y_padded + + buffer = io.BytesIO() + wavfile.write(buffer, 32000, sr_mix.astype(np.int16)) + b64 = base64.b64encode(buffer.getvalue()) + sound = "data:audio/x-wav;base64," + b64.decode("ascii") + + output_name = "TalkNet_" + str(int(time.time())) + return [sound, arpa, playback_style, output_name] + except Exception: + return [ + None, + str(traceback.format_exc()), + playback_hide, + None, + ] + + +if __name__ == "__main__": + app.run_server( + mode="external", + debug=True, + dev_tools_ui=True, + dev_tools_hot_reload=True, + threaded=True, + ) diff --git a/models/tacotron2/ControllableTalkNet/diffsvc_gui.py b/models/tacotron2/ControllableTalkNet/diffsvc_gui.py new file mode 100644 index 0000000000..837bd4e759 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/diffsvc_gui.py @@ -0,0 +1,368 @@ +import sys +import os +import io +import base64 +import dash +from jupyter_dash import JupyterDash +import dash_core_components as dcc +import dash_html_components as html +from dash.exceptions import PreventUpdate +import gdown +import traceback +from scipy.io import wavfile +import numpy as np +import torch + +sys.path.append("DiffSVC_inference_only") +from end2end import load_e2e_diffsvc, endtoend_from_path, write_to_file + +app = JupyterDash(__name__) +UPLOAD_DIRECTORY = "/content" +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" + +app.layout = html.Div( + children=[ + html.H1( + children="DiffSVC", + style={ + "font-family": "EquestriaWebfont", + "color": "#280e5f", + "font-size": "4em", + "text-align": "center", + "margin-top": "0em", + "margin-bottom": "0em", + }, + ), + html.Label("Character selection", htmlFor="speaker-dropdown"), + dcc.Dropdown( + id="speaker-dropdown", + options=[ + { + "label": "Twilight Sparkle", + "value": "Twilight", + }, + { + "label": "Discord", + "value": "Discord", + }, + { + "label": "Pinkie Pie", + "value": "Pinkie", + }, + {"label": "Nancy", "value": "Nancy"}, + ], + value=None, + style={ + "max-width": "90vw", + "width": "20em", + "margin-bottom": "0.7em", + }, + ), + html.Label( + "Upload reference audio to " + UPLOAD_DIRECTORY, + htmlFor="reference-dropdown", + ), + dcc.Store(id="current-f0s"), + dcc.Store(id="current-filename"), + dcc.Loading( + id="audio-loading", + children=[ + html.Div( + [ + html.Button( + "Update file list", + id="update-button", + style={ + "margin-right": "10px", + }, + ), + dcc.Dropdown( + id="reference-dropdown", + options=[], + value=None, + style={ + "max-width": "80vw", + "width": "30em", + }, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "row", + "margin-left": "50px", + "vertical-align": "middle", + }, + ), + html.Div( + id="audio-loading-output", + style={ + "font-style": "italic", + "margin-bottom": "0.7em", + "text-align": "center", + }, + ), + ], + type="default", + ), + html.Div( + [ + dcc.Checklist( + id="svc-options", + options=[ + {"label": "Singing mode", "value": "no_pc"}, + ], + value=[], + ) + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "row", + "margin-left": "50px", + "margin-bottom": "0.7em", + }, + ), + html.Div( + [ + html.Label("Steps", htmlFor="linsteps-input"), + dcc.Input( + id="linsteps-input", + type="number", + value="1000", + style={"width": "7em", "margin-left": "10px"}, + min=1, + max=1000, + step=1, + ), + html.Label("Step strength", htmlFor="linend-input"), + dcc.Input( + id="linend-input", + type="number", + value="0.06", + style={"width": "7em", "margin-left": "10px"}, + min=0.01, + max=0.3, + step=0.01, + ), + ], + style={ + "display": "grid", + "grid-template-columns": "auto 100px", + "margin-bottom": "0.7em", + }, + ), + dcc.Loading( + html.Div( + [ + html.Button( + "Generate", + id="gen-button", + ), + html.Audio( + id="audio-out", + controls=True, + style={ + "display": "none", + }, + ), + html.Div( + id="generated-info", + style={ + "font-style": "italic", + }, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "column", + }, + ) + ), + html.Footer( + children=""" + Presented by the Pony Preservation Project. Models by Cookie. + """, + style={"margin-top": "2em", "font-size": "0.7em"}, + ), + ], + style={ + "width": "100%", + "display": "flex", + "align-items": "center", + "justify-content": "center", + "flex-direction": "column", + "background-color": "#FFF", + }, +) + + +playback_style = { + "margin-top": "0.3em", + "margin-bottom": "0.3em", + "display": "block", + "width": "600px", + "max-width": "90vw", +} + +playback_hide = { + "display": "none", +} + + +@app.callback( + dash.dependencies.Output("reference-dropdown", "options"), + [ + dash.dependencies.Input("update-button", "n_clicks"), + ], +) +def update_filelist(n_clicks): + filelist = [] + supported_formats = [".wav", ".ogg", ".mp3", "flac"] + for x in os.listdir(UPLOAD_DIRECTORY): + if x[-4:].lower() in supported_formats: + filelist.append({"label": x, "value": x}) + return filelist + + +@app.callback( + [ + dash.dependencies.Output("audio-loading-output", "children"), + dash.dependencies.Output("current-filename", "data"), + ], + [ + dash.dependencies.Input("reference-dropdown", "value"), + ], +) +def select_file(dropdown_value): + if dropdown_value is not None: + return [ + "Selected " + dropdown_value, + dropdown_value, + ] + else: + return ["No audio selected", None] + + +def download_model(drive_id, outname): + d = "https://drive.google.com/uc?id=" + model_dir = os.path.join(UPLOAD_DIRECTORY, "models") + out_fullpath = os.path.join(model_dir, outname) + out_dir = os.path.dirname(out_fullpath) + + if not os.path.exists(model_dir): + os.mkdir(model_dir) + if not os.path.exists(out_fullpath): + os.makedirs(out_dir, exist_ok=True) + gdown.download( + d + drive_id, + out_fullpath, + quiet=False, + ) + return out_fullpath + + +svc_model, svc_name, last_end, last_step = None, None, None, None + + +@app.callback( + [ + dash.dependencies.Output("audio-out", "src"), + dash.dependencies.Output("generated-info", "children"), + dash.dependencies.Output("audio-out", "style"), + ], + [dash.dependencies.Input("gen-button", "n_clicks")], + [ + dash.dependencies.State("speaker-dropdown", "value"), + dash.dependencies.State("svc-options", "value"), + dash.dependencies.State("current-filename", "data"), + dash.dependencies.State("linend-input", "value"), + dash.dependencies.State("linsteps-input", "value"), + ], +) +def generate_audio(n_clicks, speaker, svc_options, wav_name, lin_end, lin_n_steps): + global svc_model, svc_name, last_end, last_step + + if n_clicks is None: + raise PreventUpdate + if speaker is None: + return [None, "No character selected", playback_hide] + if wav_name is None: + return [None, "No reference audio selected", playback_hide] + + try: + diffsvc_id = "1Uh17L1JtynFDgX9X6jKLPtD_t4IcTEp9" + asr_id = "1qt0pGhCbH0TltFFwSP2Oy3NClthcpr7N" + hifigan_id = "1QQT0HjMhGgDuPhyesZYbvck2lEY7GHAB" + hifigan_config_id = "1mi_O54zi6nW2eU6VuAO_Y4gv_Df-H9Yv" + + diffsvc_path = download_model( + diffsvc_id, + os.path.join(diffsvc_id, "diffsvc_model"), + ) + asr_path = download_model( + asr_id, + os.path.join(asr_id, "asr_model"), + ) + hifigan_path = download_model( + hifigan_id, + os.path.join(hifigan_id, "hifigan_model"), + ) + download_model( + hifigan_config_id, + os.path.join(hifigan_id, "config.json"), + ) + if svc_name != diffsvc_id: + svc_model = load_e2e_diffsvc( + diffsvc_path=diffsvc_path, + dilated_asr_path=asr_path, + hifigan_path=hifigan_path, + device=DEVICE, + ) + svc_name = diffsvc_id + if last_end != lin_end or last_step != lin_n_steps: + svc_model[0].generator.diffusion.set_noise_schedule( + 1e-4, float(lin_end), int(lin_n_steps), device=DEVICE + ) + last_end = lin_end + last_step = lin_n_steps + + pred_audio = endtoend_from_path( + *svc_model, + os.path.join(UPLOAD_DIRECTORY, wav_name), + speaker, + "no_pc" not in svc_options, + "no_pc" not in svc_options, + "no_pc" not in svc_options, + t_max_step=int(lin_n_steps), + ) + pred_audio *= 32768.0 + + buffer = io.BytesIO() + wavfile.write( + buffer, + svc_model[4].sampling_rate, + pred_audio.squeeze().cpu().numpy().astype(np.int16), + ) + b64 = base64.b64encode(buffer.getvalue()) + sound = "data:audio/x-wav;base64," + b64.decode("ascii") + return [sound, "Conversion complete", playback_style] + except Exception: + return [None, str(traceback.format_exc()), playback_hide] + + +if __name__ == "__main__": + app.run_server( + mode="external", + debug=True, + dev_tools_ui=True, + dev_tools_hot_reload=True, + threaded=True, + ) diff --git a/models/tacotron2/ControllableTalkNet/model_lists/sortanon_models.json b/models/tacotron2/ControllableTalkNet/model_lists/sortanon_models.json new file mode 100644 index 0000000000..3d7033c307 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/model_lists/sortanon_models.json @@ -0,0 +1,165 @@ +[ + {"source": "MLP", + "characters": [ + { + "name": "Applejack", + "drive_id": "1kpEjZ3YqMN3chKSXODOqayEm581rxj4r", + "is_singing": false + }, + { + "name": "Celestia", + "drive_id": "1whXXcnXu9XPcI60xIkTEofpaDYDOw5yB", + "is_singing": false + }, + { + "name": "Chrysalis", + "drive_id": "1bb5jKAcQcEQbx1feVwT1UmAEgocINh-E", + "is_singing": false + }, + { + "name": "Discord", + "drive_id": "1Cg9Oc_K9UDe5WgVDAcaCSbbBoo-Npj1E", + "is_singing": false + }, + { + "name": "Fluttershy", + "drive_id": "1KgVnjrnxZTXgjnI56ilkq5G4UJCbbwZZ", + "is_singing": false + }, + { + "name": "Luna", + "drive_id": "1_ztAbe5YArCMwyyQ_G9lUiz74ym5xJKC", + "is_singing": false + }, + { + "name": "Maud Pie", + "drive_id": "132G6oD0HHPPn4t1H6IkYv18_F0UVLWgi", + "is_singing": false + }, + { + "name": "Pinkie Pie", + "drive_id": "1CdYZ2r52mtgJsFs88U0ZViMSnzpQ_HRp", + "is_singing": false + }, + { + "name": "Rainbow Dash", + "drive_id": "1k3EMXxLC0fLvfxzGbeP6B6plgu9hqCSx", + "is_singing": false + }, + { + "name": "Rarity", + "drive_id": "1QWBvQSso4guc1LRUD40WRJ8DY2CfqHGK", + "is_singing": false + }, + { + "name": "Spike", + "drive_id": "1TKFdmFLttjjzByj2fZW8J70ZHjR-RTwc", + "is_singing": false + }, + { + "name": "Starlight Glimmer", + "drive_id": "1M1AMBq_xjwGTNzRUCXtSLIDJHbcSs3zR", + "is_singing": false + }, + { + "name": "Sunset Shimmer", + "drive_id": "1x1aJt06lBvzUWRlxJ9CEKcFHxQxZPpST", + "is_singing": false + }, + { + "name": "Trixie", + "drive_id": "1a3CYt0-oTTSFjxtZvAVMpClTmQteYua5", + "is_singing": false + }, + { + "name": "Twilight Sparkle", + "drive_id": "1QnOliOAmerMUNuo2wXoH-YoainoSjZen", + "is_singing": false + }, + { + "name": "Twilight Sparkle (whispering)", + "drive_id": "14_TUQVirITdyBh9etfNV8KFFhi_PUs30", + "is_singing": false + }, + { + "name": "Zecora", + "drive_id": "1gL0hqqB7952Q1S185moQd_DRCFfIa3_g", + "is_singing": false + }, + { + "name": "Twilight Sparkle (singing)", + "drive_id": "10CENYWV5ugTXZbnsldN6OKR7wkDEe7V7", + "is_singing": true + }, + { + "name": "Pinkie Pie (singing)", + "drive_id": "1lo-8wI8iB3GArrzIDlaNTA1If8WX0OEf", + "is_singing": true + }, + { + "name": "Applejack (singing)", + "drive_id": "1kegd0J2G4nogaX4IOB2O_vd8msaqMx8M", + "is_singing": true + }, + { + "name": "Fluttershy (singing)", + "drive_id": "1BBdTHis91MwnHTt7tD_xtZ-nQ9SgvqD6", + "is_singing": true + }, + { + "name": "Rarity (singing)", + "drive_id": "1EI7npfQeQYuP6b_Gy8iwA5zAg9Yj5MJA", + "is_singing": true + }, + { + "name": "Rainbow Dash (singing)", + "drive_id": "1q4RgCEVJCSaR_1Dh0rxs6JVRJt30hQRK", + "is_singing": true + }, + { + "name": "Apple Bloom", + "drive_id": "1Wm-7gqws0B3j8mtSQa77Tdk3RZwHEwT5", + "is_singing": false + }, + { + "name": "Scootaloo", + "drive_id": "1YkV1VtP1w5XOx3jYYarrCKSzXCB_FLCy", + "is_singing": false + }, + { + "name": "Sweetie Belle", + "drive_id": "1jLX0Py6j8uY93Fjf2l0HOZQYXiShfWUO", + "is_singing": false + }, + { + "name": "Cozy Glow", + "drive_id": "1cUQQ_4KXTXbmH3MqwwyZu77SpMr20HAL", + "is_singing": false + }, + { + "name": "Big McIntosh", + "drive_id": "1UzL5vo_ykmRdbi-5JOlsQzNKkoV4tJBP", + "is_singing": false + }, + { + "name": "Cadance", + "drive_id": "1w98tMJPfGhWvl797gImMfErPXDz_hA-6", + "is_singing": false + }, + { + "name": "Granny Smith", + "drive_id": "1FJuxLB6MhcHLtmEnOIxhN-yxJ85kmacu", + "is_singing": false + }, + { + "name": "Shining Armor", + "drive_id": "1PUUS71w2ik0uuycNB30nXFze8C7O8OzY", + "is_singing": false + }, + { + "name": "Tirek", + "drive_id": "1rcPDqgDeCIHGDdvfOo-fxfA1XeM4g3CB", + "is_singing": false + } + ]} +] diff --git a/models/tacotron2/ControllableTalkNet/mycroft_talknet.py b/models/tacotron2/ControllableTalkNet/mycroft_talknet.py new file mode 100644 index 0000000000..d94eb8ed04 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/mycroft_talknet.py @@ -0,0 +1,114 @@ +import os +from flask import Flask, request, send_file +import sys +import torch +import numpy as np +from scipy.io import wavfile +import io +from nemo.collections.tts.models import TalkNetSpectModel +from nemo.collections.tts.models import TalkNetPitchModel +from nemo.collections.tts.models import TalkNetDursModel +import json + +sys.path.append("hifi-gan") +from env import AttrDict +from meldataset import MAX_WAV_VALUE +from models import Generator +from denoiser import Denoiser + + +app = Flask(__name__) +RUN_PATH = os.path.dirname(os.path.realpath(__file__)) +DEVICE = "cuda:0" + + +def load_hifigan(model_name, conf_name): + # Load HiFi-GAN + conf = os.path.join("hifi-gan", conf_name + ".json") + with open(conf) as f: + json_config = json.loads(f.read()) + h = AttrDict(json_config) + torch.manual_seed(h.seed) + hifigan = Generator(h).to(torch.device(DEVICE)) + state_dict_g = torch.load(model_name, map_location=torch.device(DEVICE)) + hifigan.load_state_dict(state_dict_g["generator"]) + hifigan.eval() + hifigan.remove_weight_norm() + denoiser = Denoiser(hifigan, mode="normal") + return hifigan, h, denoiser + + +def generate_json(input, outpath): + output = "" + sample_rate = 22050 + lpath = input.split("|")[0].strip() + size = os.stat(lpath).st_size + x = { + "audio_filepath": lpath, + "duration": size / (sample_rate * 2), + "text": input.split("|")[1].strip(), + } + output += json.dumps(x) + "\n" + with open(outpath, "w", encoding="utf8") as w: + w.write(output) + + +def load_talknet(talknet_path): + with torch.no_grad(): + tnmodel = TalkNetSpectModel.restore_from(talknet_path) + durs_path = os.path.join(os.path.dirname(talknet_path), "TalkNetDurs.nemo") + tndurs = TalkNetDursModel.restore_from(durs_path) + tnmodel.add_module("_durs_model", tndurs) + pitch_path = os.path.join(os.path.dirname(talknet_path), "TalkNetPitch.nemo") + tnpitch = TalkNetPitchModel.restore_from(pitch_path) + tnmodel.add_module("_pitch_model", tnpitch) + tnmodel.eval() + return tnmodel + + +def generate_audio(transcript, tnmodel, hifigan, denoiser): + with torch.no_grad(): + tokens = tnmodel.parse(text=transcript.strip()) + spect = tnmodel.generate_spectrogram(tokens=tokens) + + y_g_hat = hifigan(spect.float()) + audio = y_g_hat.squeeze() + audio = audio * MAX_WAV_VALUE + audio_denoised = denoiser(audio.view(1, -1), strength=35)[:, 0] + audio_np = audio_denoised.detach().cpu().numpy().reshape(-1).astype(np.int16) + + buffer = io.BytesIO() + wavfile.write(buffer, 22050, audio_np) + return buffer + + +@app.route("/", methods=["GET"]) +def get_check(): + return "TalkNet server online" + + +@app.route("/api/tts", methods=["GET"]) +def get_tts(): + if "text" not in request.args: + return "" + transcript = request.args.get("text") + return send_file( + generate_audio(transcript, tnmodel, hifigan, denoiser), + attachment_filename="audio.wav", + mimetype="audio/x-wav", + ) + + +if __name__ == "__main__": + hifigan, h, denoiser = load_hifigan( + os.path.join( + RUN_PATH, "models", "1QnOliOAmerMUNuo2wXoH-YoainoSjZen", "hifiganmodel" + ), + "config_v1", + ) + tnmodel = load_talknet( + os.path.join( + RUN_PATH, "models", "1QnOliOAmerMUNuo2wXoH-YoainoSjZen", "TalkNetSpect.nemo" + ) + ) + app.run(debug=False) diff --git a/models/tacotron2/ControllableTalkNet/requirements.txt b/models/tacotron2/ControllableTalkNet/requirements.txt new file mode 100644 index 0000000000..f7270b68cb --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/requirements.txt @@ -0,0 +1,28 @@ +tensorflow==2.4.1 +tensorboard==2.4.1 +dash==1.21.0 +dash-bootstrap-components==0.13.0 +jupyter-dash==0.4.0 +psola==0.0.1 +wget==3.2 +unidecode==1.2.0 +pysptk==0.1.18 +frozendict==2.0.3 +torch==1.8.1+cu111 +torchvision==0.9.1+cu111 +torchaudio==0.8.1 +torchtext==0.9.1 +torch_stft==0.1.4 +kaldiio==2.17.2 +pydub==0.25.1 +pyannote.audio==1.1.2 +g2p_en==2.1.0 +pesq==0.0.2 +pystoi==0.3.3 +crepe==0.0.12 +resampy==0.2.2 +ffmpeg-python==0.2.0 +tqdm +gdown==3.13.0 +editdistance==0.5.3 +ipywidgets==7.6.3 diff --git a/models/tacotron2/ControllableTalkNet/talknet_offline.py b/models/tacotron2/ControllableTalkNet/talknet_offline.py new file mode 100644 index 0000000000..b2b37a6499 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/talknet_offline.py @@ -0,0 +1,17 @@ +if __name__ == "__main__": + import os + from controllable_talknet import * + + if os.path.exists("/talknet/is_docker"): + app.run_server( + host="0.0.0.0", + mode="external", + debug=False, + threaded=True, + ) + else: + app.run_server( + mode="external", + debug=False, + threaded=True, + ) diff --git a/models/tacotron2/ControllableTalkNet/talknet_singer.py b/models/tacotron2/ControllableTalkNet/talknet_singer.py new file mode 100644 index 0000000000..34f573dca7 --- /dev/null +++ b/models/tacotron2/ControllableTalkNet/talknet_singer.py @@ -0,0 +1,225 @@ +from collections import OrderedDict +from typing import List + +import torch +from hydra.utils import instantiate +from omegaconf import DictConfig +from pytorch_lightning import Trainer +from torch import nn +from torch.nn import functional as F +import numpy as np + +from nemo.collections.asr.data.audio_to_text import AudioToCharWithDursF0Dataset +from nemo.collections.tts.helpers.helpers import get_mask_from_lengths +from nemo.collections.tts.models.base import SpectrogramGenerator +from nemo.collections.tts.modules.talknet import ( + GaussianEmbedding, + MaskedInstanceNorm1d, + StyleResidual, +) +from nemo.core import Exportable +from nemo.core.classes import ModelPT, PretrainedModelInfo, typecheck +from nemo.core.neural_types import MelSpectrogramType, NeuralType + + +class TalkNetSingerModel(SpectrogramGenerator, Exportable): + """TalkNet's mel spectrogram prediction pipeline.""" + + @property + def output_types(self): + return OrderedDict( + {"mel-spectrogram": NeuralType(("B", "D", "T"), MelSpectrogramType())} + ) + + def __init__(self, cfg: DictConfig, trainer: "Trainer" = None): + super().__init__(cfg=cfg, trainer=trainer) + typecheck.set_typecheck_enabled(enabled=False) + + cfg = self._cfg + self.vocab = AudioToCharWithDursF0Dataset.make_vocab( + **cfg.train_ds.dataset.vocab + ) + self.blanking = cfg.train_ds.dataset.blanking + self.preprocessor = instantiate(cfg.preprocessor) + self.embed = GaussianEmbedding(self.vocab, cfg.d_char) + self.norm_f0 = MaskedInstanceNorm1d(1) + self.res_f0 = StyleResidual(cfg.d_char, 1, kernel_size=3) + self.model = instantiate(cfg.model) + d_out = cfg.model.jasper[-1].filters + self.proj = nn.Conv1d(d_out, cfg.n_mels, kernel_size=1) + + def forward(self, text, text_len, durs, f0): + x, x_len = self.embed(text, durs).transpose(1, 2), durs.sum(-1) + f0_sin = np.nan_to_num(np.sin(9.06472 * np.log(0.0363636 * f0.cpu()))) + f0_cos = np.nan_to_num(np.cos(9.06472 * np.log(0.0363636 * f0.cpu()))) + f0_note = torch.stack( + (torch.from_numpy(f0_sin), torch.from_numpy(f0_cos)), 1 + ).to(self.device) + f0, f0_mask = f0.clone(), f0 > 0.0 + f0 = self.norm_f0(f0.unsqueeze(1), f0_mask) + f0[~f0_mask.unsqueeze(1)] = 0.0 + x = self.res_f0(x, f0) + x = torch.cat((x, f0_note), 1) + y, _ = self.model(x, x_len) + mel = self.proj(y) + return mel + + @staticmethod + def _metrics(true_mel, true_mel_len, pred_mel): + loss = F.mse_loss(pred_mel, true_mel, reduction="none").mean(dim=-2) + mask = get_mask_from_lengths(true_mel_len) + loss *= mask.float() + loss = loss.sum() / mask.sum() + return loss + + def training_step(self, batch, batch_idx): + audio, audio_len, text, text_len, durs, f0, f0_mask = batch + mel, mel_len = self.preprocessor(audio, audio_len) + pred_mel = self(text=text, text_len=text_len, durs=durs, f0=f0) + loss = self._metrics(true_mel=mel, true_mel_len=mel_len, pred_mel=pred_mel) + train_log = {"train_loss": loss} + return {"loss": loss, "progress_bar": train_log, "log": train_log} + + def validation_step(self, batch, batch_idx): + audio, audio_len, text, text_len, durs, f0, f0_mask = batch + mel, mel_len = self.preprocessor(audio, audio_len) + pred_mel = self(text=text, text_len=text_len, durs=durs, f0=f0) + loss = self._metrics(true_mel=mel, true_mel_len=mel_len, pred_mel=pred_mel) + val_log = {"val_loss": loss} + self.log_dict( + val_log, prog_bar=False, on_epoch=True, logger=True, sync_dist=True + ) + + @staticmethod + def _loader(cfg): + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader( # noqa + dataset=dataset, + collate_fn=dataset.collate_fn, + **cfg.dataloader_params, + ) + + def setup_training_data(self, cfg): + self._train_dl = self._loader(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self._loader(cfg) + + def setup_test_data(self, cfg): + """Omitted.""" + pass + + def parse(self, text: str, **kwargs) -> torch.Tensor: + return torch.tensor(self.vocab.encode(text)).long().unsqueeze(0).to(self.device) + + def generate_spectrogram(self, tokens: torch.Tensor, **kwargs) -> torch.Tensor: + assert hasattr(self, "_durs_model") and hasattr(self, "_pitch_model") + + if self.blanking: + tokens = [ + AudioToCharWithDursF0Dataset.interleave( + x=torch.empty(len(t) + 1, dtype=torch.long, device=t.device).fill_( + self.vocab.blank + ), + y=t, + ) + for t in tokens + ] + tokens = AudioToCharWithDursF0Dataset.merge( + tokens, value=self.vocab.pad, dtype=torch.long + ) + + text_len = torch.tensor(tokens.shape[-1], dtype=torch.long).unsqueeze(0) + durs = self._durs_model(tokens, text_len) + durs = durs.exp() - 1 + durs[durs < 0.0] = 0.0 + durs = durs.round().long() + + # Pitch + f0_sil, f0_body = self._pitch_model(tokens, text_len, durs) + sil_mask = f0_sil.sigmoid() > 0.5 + f0 = f0_body * self._pitch_model.f0_std + self._pitch_model.f0_mean + f0 = (~sil_mask * f0).float() + + # Spect + mel = self(tokens, text_len, durs, f0) + + return mel + + def forward_for_export(self, tokens: torch.Tensor, text_len: torch.Tensor): + durs = self._durs_model(tokens, text_len) + durs = durs.exp() - 1 + durs[durs < 0.0] = 0.0 + durs = durs.round().long() + + # Pitch + f0_sil, f0_body = self._pitch_model(tokens, text_len, durs) + sil_mask = f0_sil.sigmoid() > 0.5 + f0 = f0_body * self._pitch_model.f0_std + self._pitch_model.f0_mean + f0 = (~sil_mask * f0).float() + + # Spect + x, x_len = self.embed(tokens, durs).transpose(1, 2), durs.sum(-1) + f0_sin = np.nan_to_num(np.sin(9.06472 * np.log(0.0363636 * f0.cpu()))) + f0_cos = np.nan_to_num(np.cos(9.06472 * np.log(0.0363636 * f0.cpu()))) + f0_note = torch.stack( + (torch.from_numpy(f0_sin), torch.from_numpy(f0_cos)), 1 + ).to(self.device) + f0, f0_mask = f0.clone(), f0 > 0.0 + f0 = self.norm_f0(f0.unsqueeze(1), f0_mask) + f0[~f0_mask.unsqueeze(1)] = 0.0 + x = self.res_f0(x, f0) + x = torch.cat((x, f0_note), 1) + y, _ = self.model(x, x_len) + mel = self.proj(y) + + return mel + + def force_spectrogram( + self, tokens: torch.Tensor, durs: torch.Tensor, f0: torch.Tensor, **kwargs + ) -> torch.Tensor: + if self.blanking: + tokens = [ + AudioToCharWithDursF0Dataset.interleave( + x=torch.empty(len(t) + 1, dtype=torch.long, device=t.device).fill_( + self.vocab.blank + ), + y=t, + ) + for t in tokens + ] + tokens = AudioToCharWithDursF0Dataset.merge( + tokens, value=self.vocab.pad, dtype=torch.long + ) + + text_len = torch.tensor(tokens.shape[-1], dtype=torch.long).unsqueeze(0) + durs_len = torch.tensor(durs.shape[-1], dtype=torch.long).unsqueeze(0) + assert text_len == durs_len + + # Spect + mel = self(tokens, text_len, durs, f0) + return mel + + @classmethod + def list_available_models(cls) -> "List[PretrainedModelInfo]": + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_en_talknet", + location=( + "https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_talknet/versions/1.0.0rc1/files" + "/talknet_spect.nemo" + ), + description=( + "This model is trained on LJSpeech sampled at 22050Hz, and can be used to generate female " + "English voices with an American accent." + ), + class_=cls, # noqa + aliases=["TalkNet-22050Hz"], + ) + list_of_models.append(model) + return list_of_models