Merge remote-tracking branch 'old_f/master'

This commit is contained in:
Brandon Thomas committed 2022-04-03 02:39:04 -04:00
commit 23602137a8
125 files changed
+147481

No files matched your search

+16
View File
@@ -0,0 +1,16 @@
*~
.idea/libraries/Cargo__trumpet_.xml
.idea/workspace.xml
bower_components/
docker/*zip
docker/files/
node_modules/
sounds/
target/
web/*.css
web/*.js
web/deploy/
web/images/
web/output/
brandon.audio/deploy/
brandon.audio/output/
+9
View File
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectCodeStyleSettingsManager">
<option name="PER_PROJECT_SETTINGS">
<value />
</option>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default (1)" />
</component>
</project>
+22
View File
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<resourceExtensions />
<wildcardResourcePatterns>
<entry name="!?*.java" />
<entry name="!?*.form" />
<entry name="!?*.class" />
<entry name="!?*.groovy" />
<entry name="!?*.scala" />
<entry name="!?*.flex" />
<entry name="!?*.kt" />
<entry name="!?*.clj" />
<entry name="!?*.aj" />
</wildcardResourcePatterns>
<annotationProcessing>
<profile default="true" name="Default" enabled="false">
<processorPath useClasspath="true" />
</profile>
</annotationProcessing>
</component>
</project>
+3
View File
@@ -0,0 +1,3 @@
<component name="CopyrightManager">
<settings default="" />
</component>
+10
View File
@@ -0,0 +1,10 @@
<component name="ProjectDictionaryState">
<dictionary name="bt">
<words>
<w>arpabet</w>
<w>monophones</w>
<w>polyphone</w>
<w>polyphones</w>
</words>
</dictionary>
</component>
+13
View File
@@ -0,0 +1,13 @@
<component name="libraryTable">
<library name="Rust &lt;trumpet&gt;">
<CLASSES>
<root url="jar://$USER_HOME$/lib/rust-1.12.1-src.zip!/rust-1.12.1/src/libstd" />
<root url="jar://$USER_HOME$/lib/rust-1.12.1-src.zip!/rust-1.12.1/src/libcore" />
<root url="jar://$USER_HOME$/lib/rust-1.12.1-src.zip!/rust-1.12.1/src/libcollections" />
<root url="jar://$USER_HOME$/lib/rust-1.12.1-src.zip!/rust-1.12.1/src/liballoc" />
<root url="jar://$USER_HOME$/lib/rust-1.12.1-src.zip!/rust-1.12.1/src/librustc_unicode" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</component>
+15
View File
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CargoProjects">
<cargoProject FILE="$PROJECT_DIR$/Cargo.toml" />
</component>
<component name="EntryPointsManager">
<entry_points version="2.0" />
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_3" default="true">
<output url="file://$PROJECT_DIR$/out" />
</component>
<component name="RustProjectSettings">
<option name="toolchainHomeDirectory" value="$USER_HOME$/.cargo/bin" />
</component>
</project>
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/trumpet.iml" filepath="$PROJECT_DIR$/trumpet.iml" />
</modules>
</component>
</project>
Generated
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>
+48
View File
@@ -0,0 +1,48 @@
Things that break
=================
On A.I.,
and @GovGaryJohnson.
RT @Cole_No_Slaw:
Dems' plans
Dem
only $29.99!
Thank you to our U.S. Navy
#TheGreaterGood.
at 2:00pm!
60-66%
140K
w/ corporate
Times
-----
2:00pm two p m
7:30pm seven thirty p m
5:30ET five thirty eastern time
9pm ET nine p m eastern time
8pm ET eight p m eastern time
8:00 P.M. eight p m
9 p.m. ET nine p m eastern time
units:
- AM, a.m.
- et, pt, ET, PT
2:00pm -> 2:00 pm
7:30p.m. -> 7:30 p.m.
9pm -> 9 pm
8:00pm -> 8:00 pm
5:30ET
9pm ET
8pm ET
8:00 P.M.
9 p.m. ET
TimeWord
- am, a.m., AM, A.M.
- pm, p.m., PM, P.M.
- ET, PT
\d:\d+\s?[(am)|(pm)]
Generated
+1160
View File
File diff suppressed because it is too large. Load diff
+31
View File
@@ -0,0 +1,31 @@
[package]
name = "trumpet"
version = "0.12.0"
authors = [ "Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>" ]
[[bin]]
name = "trumpet"
[dependencies]
#cpal = "*" # Hardware audio output (for testing); works well with Hound.
arpabet = "0.2"
chrono = { version = "0.2.*", features = ["rustc-serialize"] }
clap = "1.5.*" # CLI parsing.
hound = "2.0.*" # WAV codec.
hyper = "0.9.*"
iron = "0.4.*"
lazy_static = "0.2.*"
log = "0.3.*"
mount = "0.2.*" # Path mounting middleware for Iron
params = "0.3.*"
regex = "0.1.*"
resolve = "0.1.1"
router = "0.2.*"
rust-crypto = "0.2.*"
rustc-serialize = "0.3.15"
staticfile = "0.3.0" # Iron static file serving plugin
time = "0.1.*"
toml = "0.1.*"
urlencoded = "0.4.*"
+12
View File
@@ -0,0 +1,12 @@
- EY1 as in dangerous (D EY1 N JH ER0 AH0 S)
- K as in killing (K IH1 L IH0 NG)
- IH1 as in killing (K IH1 L IH0 NG)
- IH0 as in convict (K AA1 N V IH0 K T)
Fix problems:
-------------
Clinton
Right
1M lives
November 8,
probe.Such a dishonest person
+256
View File
@@ -0,0 +1,256 @@
Trumpet
=======
1. for the lulz
2. ???
3. lol
Namecheap DNS Notes
-------------------
### `trumped.com` DNS
```
Type Host Value TTL
CNAME @ `trumped.com.s3-website-us-east-1.amazonaws.com.` 5 MIN
CNAME api `junglehorseapi-env.us-west-2.elasticbeanstalk.com.` 5 MIN
CNAME cdn `trumped-frontend.s3.amazonaws.com.` 5 MIN
CNAME www `trumped-frontend.s3.amazonaws.com.` 5 MIN
URL * `http://trumped.com` Unmasked
```
### `brandon.audio` DNS
```
Type Host Value TTL
CNAME @ `brandon.audio.s3-website-us-east-1.amazonaws.com` 5 MIN
CNAME api `junglehorseapi-env.us-west-2.elasticbeanstalk.com.` 5 MIN
CNAME cdn `cdn.brandon.audio.s3.amazonaws.com.` 5 MIN
CNAME www `cdn.brandon.audio.s3.amazonaws.com.` 5 MIN
URL * `http://brandon.audio` Unmasked
```
### Old `jungle.horse`
```
Type Host Value TTL
CNAME @ `junglehorse-frontend.s3-website-us-east-1.amazonaws.com.` 5 MIN
CNAME api `junglehorseapi-env.us-west-2.elasticbeanstalk.com.` 20 MIN
CNAME cdn `dy6kf1ub3ccan.cloudfront.net.` 20 MIN
URL www `http://jungle.horse` 301 Permanent
```
AWS Notes
---------
### S3 Bucket
- Bucket *must* be named per the domain, eg. `trumped.com` instead of `trumped-index`.
- Static website hosting: set index.html, error.html
- Set the Bucket Policy: ```{
"Version": "2012-10-17",
"Id": "Policy1535378813796",
"Statement": [
{
"Sid": "Stmt1478413958263",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::961528074194:user/junglehorse"
},
"Action": "s3:*",
"Resource": "arn:aws:s3:::trumped.com/*"
},
{
"Sid": "PublicReadGetObject",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::trumped.com/*"
}
]
}
```
- Set CORS headers,
```xml
<?xml version="1.0" encoding="UTF-8"?>
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<CORSRule>
<AllowedOrigin>*</AllowedOrigin>
<AllowedMethod>GET</AllowedMethod>
<AllowedMethod>HEAD</AllowedMethod>
<MaxAgeSeconds>3000</MaxAgeSeconds>
<AllowedHeader>*</AllowedHeader>
</CORSRule>
</CORSConfiguration>
```
- Note that the URL is `http://trumped.com.s3-website-us-east-1.amazonaws.com/` (static websites),
not `http://trumped.com.s3.amazonaws.com/` (REST endpoints)!
### Cloudfront
- Origin domain name: trumped-frontend.s3.amazonaws.com
- Origin path: (null)
- Origin id: S3-trumped-frontend
- Restrict access: no
- Alternative domain names (CNAMEs): trumped.com, www.trumped.com, cdn.trumped.com
- SSL certificate: Default Cloudfront (\*.cloudfront.net)
- Default root object: index.html
- Extra:
- Behaviors: `/` at precedence 0, forward query strings.
- Error pages (403, 404) to direct to `/error.html`
### Elastic Beanstalk
- (Make sure you're in the same region you set the app up in!)
- https://us-west-2.console.aws.amazon.com/elasticbeanstalk/home?region=us-west-2#/environment/dashboard?applicationName=junglehorse-api&environmentId=e-bgpvm79qkt
Running the server
------------------
### Locally
1. Install Rust >= 1.1.0
2. `cargo run`
### On the Remote Dedicated Server (non-AWS)
1. Install and configure nginx as a reverse HTTP proxy.
2. SCP Rust server app binaries and start on required ports, ie.
`trumpet -p PORT`.
Before moving to AWS, the DNS A records pointed to `209.239.112.74`,
which is the hostname `colossus960.startdedicated.com`.
### On AWS Infrastructure.
1. The frontend is hosted on AWS + Cloudfront.
2. The backend is hosted on Elastic Beanstalk (EBS).
The AWS setup is divided into two buckets: one that houses simply index.html,
which is hosted from `jungle.horse`. The other assets are distributed via
Cloudfront, and are loaded from `cdn.jungle.horse`. There is some CORS header
setup work to do. (See the s3-cors and s3-bucket files for config.)
The backend is a docker image containing the compiled Rust binary, config file,
and sound files. It can be built into a zip with the `./ebs-deployable` script.
This is then uploaded via the EBS panel's front page (the "Upload and Deploy"
button). This is hosted from `api.jungle.horse`.
Docker Notes
------------
Some notes for local management and testing of Docker images:
Build docker image from Dockerfile (in same working directory):
`sudo docker build -t TAGNAME .`
Run docker image interactively with port forwarding to host machine:
`sudo docker run -t -p 8000:8000 -i TAGNAME /bin/bash`
Misc Notes on Speech
--------------------
### Phonemes
- [CMU Dictionary](http://svn.code.sf.net/p/cmusphinx/code/trunk/cmudict/),
which uses [Arpabet](https://en.wikipedia.org/wiki/Arpabet).
- 54 units in the Arpabet:
- Vowels:
- 10 monophthongs (AO, AA, IY, UW, EH, IH, UH, AH, AX, AE)
- 5 diphthongs (EY, AY, OW, AW, OY)
- 8 R-colored vowels (ER, AXR, EH R, UH R, AO R, AA R, IH|IY R, AW R)
- Consonants:
- 6 Stops (P, B, T, D, K, G)
- 2 Affricates (CH, JH)
- 9 Fricatives (F, V, TH, DH, S, Z, SH, ZH, HH)
- 6 Nasals (M, EM, N, EN, NG, ENG)
- 5 Liquids (L, EL, R, DX, MX)
- 3 Semivowels (Y, W, Q)
### Diphones
TODO: Generate a list of phoneme 2-tuples.
### Unit Selection
- Combine multiple sources into final output (words, phonemes, diphones)
- Have alternatives (words, phonemes) that pair better with others.
- Optimize joining on a score (start/ending pitch, tempo, etc.)
Production Server
-----------------
The production server is `209.239.112.74`.
I'm using nginx as a reverse HTTP proxy.
To manage the process, `sudo service nginx restart`, etc.
To see what processes bind a port, `sudo lsof -i:80`, etc.
### Current nginx config
In sites-available, at `/etc/nginx/sites-enabled/default`,
```
server {
# XXX: Note - port 9000 is a temporary artifact; normally port 80.
listen 9000;
server_name jungle.horse;
location / {
proxy_pass http://127.0.0.1:8888;
}
}
server {
listen 9000;
server_name capitalism.store;
location / {
proxy_pass http://127.0.0.1:9999;
}
}
```
### Old IPTABLES Config
This was the old iptables-based routing. Even after flushing the rules,
port 80 still forwards to 9000 for some reason.
The rules were (prior to flush),
```
bt@colossus960:~$ sudo iptables --list-rules
-P INPUT ACCEPT
-P FORWARD ACCEPT
-P OUTPUT ACCEPT
-A INPUT -i eth0 -p tcp -m tcp --dport 80 -j ACCEPT
-A INPUT -i eth0 -p tcp -m tcp --dport 9000 -j ACCEPT
```
And,
```
bt@colossus960:~$ sudo iptables -L
Chain INPUT (policy ACCEPT)
target prot opt source destination
ACCEPT tcp -- anywhere anywhere tcp dpt:http
ACCEPT tcp -- anywhere anywhere tcp dpt:9000
Chain FORWARD (policy ACCEPT)
target prot opt source destination
Chain OUTPUT (policy ACCEPT)
target prot opt source destination
```
### User Creation
1. sudo useradd $NAME
2. sudo mkdir /home/$NAME
3. (copy .bashrc, .profile, .bash\_logout to homedir)
4. (chown homedir to new user)
5. sudo chsh -s /bin/bash $NAME
6. sudo passwd $NAME
+149
View File
@@ -0,0 +1,149 @@
Trumped TODO
============
Improve synthesis system
------------------------
- Speech modes:
- "SHOUT" words for emphasis.
- Question intonation in interrogative sentences.
- Clear conjection words: "But I think..." vs. "Blah blah, BUT blah blah blah."
- Arpabet where the numbers are parameters: AW(0), AW(2), etc.
- Multiple word samples with unit selection
- Pluralization
Words to Fix
------------
- you're / your is 'yer'. too short.
- want is 'wan'. too short.
- that's
- 'what' is 'wet'. bad.
# Needed words
- need
- shut up
- suck/sucks
- tell
- idea
# Shout words
- die
- favorite
- hands
- tiny
# Phonemes to fix
- BReak
- badASS
- deMAND
- naME
- thinKING
- unfortuNate
OLD Last Minute Burndown
------------------------
- Disable volume and speed parameters.
- In-Memory LRU Cache
- Better audio samples
- Multiple "umm" / "uhh" samples
- Fix words that sound bad
- User input filtering
- Strip spaces in frontend.
- Make sure there's no XSS injection.
- Remove speed query parameter (for now).
- Frontend filesize
- Downscale the animation. Maybe make a sprite sheet?
OLD Super Low Priority
----------------------
- Frontend filesize:
- Progressive loading of the animation (involved).
- Zepto.js
- Remove pixi.js dependency
OLDER Old Goals
---------------
*HIGH LEVEL GOALS*
1. Sound better
2. Handle load
3. Recoup costs
4. Look good (UI, speaking capability) - good reflection
5. Good code
Audio Samples
- Remove extra space between words (using parsing to insert spacing)
- Phonemes
- Real versions of ['AO0', 'AW0', 'AW2', 'AY0', 'EY0', 'OY0', 'UH0'], which
were backfilled with fakes.
Scaling
1. Load test
2. Read wave files into memory
3. Multi-server, with nginx as load balancer
4. Prevent others from calling it
5. Caching strategy
- Caching headers (make sure they work)
- Filter strings clientside to boost savings.
Tokenizing
- Plurals
- Times (9 p.m. ET)
Parsing
- (Everything in tokenizing)
- Decimal numbers
- Collapse pauses, eg: "Foo,,," -> "Foo (breath)"
- Numbers that are years (2016)!
- Twitter hashtags, usernames - conjoined word parsing (difficult)
Synthesis
- Get rid of static with "Uhhh", "Ummm" of variable length (option).
- Syllable System:
- http://cdmckay.org/blog/2012/08/15/counting-syllables-and-detecting-rhyme-in-php/
- http://stackoverflow.com/questions/33111685/convert-arpabet-to-ipa-with-stress
- Or scrape something:
- http://www.syllablecount.com/syllables/biologically
Cleanup
- Refactor into a very well-defined "SpeakRequest" with all
parameters, including new ones like "is_twitter",
"missing_word_mode = slience, record_pop, umm", etc.
- Remove old query parameters
- Set default volume
Server
1. Favicon, .ico and apple icon
2. robots.txt
Hardening
1. Reduce API surface
2. Limit length of input sentences
3. Rate limiting
Presentation
- Better frontend UI
- Testing page (current frontend)
Monetization
- Can I get sued?
- Ads
- Native Apps
- (I don't have time to learn this.)
- Contact other developers?
- PhoneGap? (Blah)
Extra
- Clientside speed changing
- Let people type nonsense syllables
OLD Parsing Examples
--------------------
- Camel case even outside of hashtags:
- "Google acquires FameBit", StateOfW, AdaLovelaceDay
- Hyphenated: 2-party
- They will get horrible sentences, violently.We have the support.#RepealThe1st
- #100YearsStrong
+22
View File
@@ -0,0 +1,22 @@
{
"name": "trumpet",
"moduleType": [],
"license": "none",
"private": true,
"ignore": [
"**/.*",
"node_modules",
"bower_components",
"web/components",
"test",
"tests"
],
"dependencies": {
"jquery": "~2.2.0",
"underscore": "~1.8.3",
"urijs": "uri.js#~1.17.0",
"SoundJS": "~0.6.2",
"buzz": "~1.1.10",
"wavesurfer.js": "^1.1.12"
}
}
+10
View File
@@ -0,0 +1,10 @@
<!doctype html>
<html lang="en-US">
<head>
<title>Brandon Audio &mdash; Error</title>
</head>
<body>
<h1>Bad Error</h1>
<p>This thing doesn't exist.</p>
</body>
</html>
+78
View File
@@ -0,0 +1,78 @@
<!doctype html>
<html lang="en-US">
<head>
<title>[brandon.audio]</title>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8">
<meta name="viewport"
content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no">
<meta name="description" content="brandon.audio">
<link rel="stylesheet" href="/assets/main.css" type="text/css">
<link href="https://fonts.googleapis.com/css?family=Fira+Sans" rel="stylesheet">
<meta name="content_hash" content="asset_content_hash">
<meta name="environment" content="DEVELOPMENT">
<meta name="api_host" content="API_HOST">
<meta name="cdn_host" content="CDN_HOST">
</head>
<body>
<div id="animation"></div>
<div id="logo">
<h1>brandon.audio</h1>
</div>
<div id="main">
<form>
<div id="input-box">
<input id="jungle" name="jungle" autocomplete="off">
</div>
<div id="buttons">
<div id="button-left">
<button id="speak">Speak</button>
</div>
<div id="button-right">
<button id="clear">Clear</button>
</div>
<div class="clear"></div>
</div>
</form>
</div>
<aside class="hidden">
<div id="about">
<a href="#" id="close_top">[close]</a>
<h2>doop</h2>
<strong>darp</strong>
<p>yes</p>
<strong>dap</strong>
<p>no</p>
<a href="#" id="close_bottom">[close]</a>
</div>
</aside>
<footer>
<div id="footer-text">
<p>
Made in Atlanta by
<a href="https://twitter.com/echelon">@echelon</a>.
</p>
<p>
By using this, you agree to
<a id="info" href="#">the things</a>.
</p>
</dvi>
</footer>
<script src="/assets/output/main.built.js"/></script>
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-32811180-3', 'auto');
ga('send', 'pageview');
</script>
</body>
</html>
+237
View File
@@ -0,0 +1,237 @@
html,
body {
margin: 0;
padding: 0;
}
body,
input,
button {
font-family: "Fira Sans";
}
#main {
margin: auto;
max-width: 1000px;
}
h1 {
color: #333;
font-family: "Happy Monkey";
font-size: 3em;
margin: 20px 0 10px;
text-align: center;
}
#animation canvas {
display: block;
margin: auto;
}
#logo {
display: none;
z-index: 100;
position: absolute;
top: 0;
width: 100%;
}
#input-box {
box-sizing: border-box;
padding: 12px 15px;
width: 100%;
}
pre {
display: inline;
}
input {
box-sizing: border-box;
font-size: 1.2em;
border: 1px solid #999;
padding: 0.3em;
width: 100%;
}
button {
background-color: #e5e5e5;
border: 0;
color: #777;
font-size: 1.2em;
margin: 20px;
min-width: 150px;
padding: 0.75em;
padding: 10px;
}
#buttons {
margin-top: 20px;
width: 100%;
}
#button-left {
float: left;
width: 50%;
}
#button-right {
float: right;
width: 50%;
}
button#speak {
display: block;
margin: 0 auto;
}
button#clear {
display: block;
margin: 0 auto;
}
a:link, a:visited {
color: #c75;
font-weight: bold;
}
aside {
position: absolute;
top: 0;
z-index: 100;
background-color: rgba(255,255,255,0.85);
width: 100%;
}
aside #about {
padding: 20px 20px 50px;
margin: auto;
max-width: 1000px;
}
#close_top,
#close_bottom {
float: right;
}
.hidden {
display: none;
}
footer {
width: 100%;
position: absolute;
bottom: 0em;
overflow: auto;
padding: 0;
margin: 0;
color: #777;
}
footer a:link,
footer a:visited {
color: #d86;
font-weight: bold;
}
#footer-text {
text-align: center;
padding: 15px;
}
.clear {
clear: both;
}
/** RESPONSIVENESS */
@media screen and (min-width: 600px) {
button {
min-width: 250px;
}
}
@media screen and (min-width: 1px) {
#animation canvas {
width: 200px;
}
#animation {
height: 117px;
}
}
@media screen and (min-width: 300px) {
#animation canvas {
width: 300px;
}
#animation {
height: 176px;
}
}
@media screen and (min-width: 400px) {
#animation canvas {
width: 400px;
}
#animation {
height: 235px;
}
}
@media screen and (min-width: 450px) {
#animation canvas {
width: 450px;
}
#animation {
height: 264px;
}
}
@media screen and (min-width: 500px) {
#animation canvas {
width: 500px;
}
#animation {
height: 293px;
}
}
@media screen and (min-width: 600px) {
#animation canvas {
width: 600px;
}
#animation {
height: 352px;
}
}
@media screen and (min-width: 800px) {
#animation canvas {
width: 800px;
}
#animation {
height: 470px;
}
}
@media screen and (min-width: 1000px) {
#animation canvas {
width: 800px;
}
#animation {
height: 470px;
}
}
/** Hide footer when using onscreen keyboard. */
@media screen and (max-height: 450px) {
footer {
display: none;
}
}
@media screen and (max-height: 740px) {
footer p {
display: inline;
}
}
+2
View File
@@ -0,0 +1,2 @@
User-agent: *
Allow: /
+104
View File
@@ -0,0 +1,104 @@
h1 {
font-family: ubuntu, tahoma;
font-size: 1.8em;
text-align: center;
color: #333;
}
input {
border: 1px solid #ccc;
padding: 8px;
margin-bottom: 1em;
}
input#text {
width: 70%;
}
input#vol {
width: 30px;
}
button {
background-color: #eee;
border: 1px solid #ccc;
padding: 8px;
}
select {
background-color: #eee;
border: 1px solid #ccc;
padding: 7px;
}
#control_elements {
display: inline;
}
@media (max-width: 600px) {
#control_elements {
display: block;
}
}
#main {
margin: auto;
width: 90%;
text-align: center;
}
.error input#text {
border-color: #f55;
background-color: #fee;
color: #500;
}
.ok input#text {
border-color: #5a5;
background-color: #efe;
}
#typeahead {
color: #777;
font-style: italic;
width: 100%;
height: 1em;
margin: 0 0 10px 0;
overflow: hidden;
/*text-overflow: ellipsis;*/
}
#typeahead ul {
list-style-type: none;
margin: 0;
padding: 0;
}
#typeahead li {
display: inline;
padding: 5px;
border: 1px;
}
#stats {
margin: 20px 0;
font-size: 0.8em;
color #777;
}
ul#stats {
list-style-type: none;
}
a:link, a:visited {
color: #00f;
text-decoration: none;
}
a:active, a:hover {
color: #00f;
text-decoration: underline;
}
+36
View File
@@ -0,0 +1,36 @@
# Configs for the server.
[default] # Production
use_caching_headers = true
sound_path = "./sounds"
html_path = "./web"
phoneme_dictionary_file = "./dictionary/cmudict-0.7b"
arpabet_override_dictionary_file = "./dictionary/cmu-changeset"
extra_dictionary_file = "./dictionary/extra-dict"
square_dictionary_file = "./dictionary/square-dict"
abbreviation_file = "./dictionary/abbreviations.dict"
# Input control. Respond with 400 when over limit.
max_num_characters = 255
max_num_words = 100
# Logging verbosity
log_handler = false
log_parsing = false
log_raw_sentence = true
log_syllable_decomposition = false
log_synthesis = false
log_timing = true
[x16] # Local Development
use_caching_headers = false
sound_path = "/home/bt/Dropbox/jungle.horse/sounds"
# Logging verbosity
log_handler = false
log_parsing = true
log_raw_sentence = true
log_syllable_decomposition = true
log_synthesis = true
log_timing = false
+26
View File
@@ -0,0 +1,26 @@
#!/bin/bash
# This creates a Dockerfile in the AWS EBS format that is ready for deployment.
# Dockerfile, Dockerrun.aws.json, and packaged artifact files.
# In this case, the Rust server is packaged with the other dependencies.
set -ex
declare -r artifact_dir_name="files/"
declare -r artifact_dir="./docker/${artifact_dir_name}"
# Build the thing.
cargo build --release
# Copy binary, sound samples, and configs (only if updated).
# No need to copy web files as these are hosted on AWS/EC2 instead.
mkdir -p $artifact_dir
rsync -utrv ./target/release/trumpet $artifact_dir
rsync -utrv ~/Dropbox/jungle.horse/sounds $artifact_dir
rsync -utrv {dictionary,config.toml} $artifact_dir
declare -r artifact_zip="./docker_image.zip"
pushd docker
zip -r $artifact_zip Dockerfile Dockerrun.aws.json
zip -r $artifact_zip $artifact_dir_name
popd
+18
View File
@@ -0,0 +1,18 @@
These are great examples of synthesis
-------------------------------------
bad
because
country
crucified
debates
delicious
disqualifies
honestly
inside
outside
patience
people
righteous
satchell
savior
things
+122
View File
@@ -0,0 +1,122 @@
These words need improvement
----------------------------
"^^ bill's
"^^ hillary's
"^^ obama's
"^^^ again (BAD)
"^^^ by
"^^^ by (BAD)
"^^^ ha, haha, hahaha"
"^^^ how (THIS IS SO BAD)
"^^^ rigging
"^^^ then (BAD)
"do you?"
"for you"
"really?"
"to do"
4chan
access
again
america (non-shout)
arab
arabic
aren't
ask
asswipe
bear
being
beta
bin
bite
bullshit
buy
can
cat
chinese
code
cunt
dank
derp
dildo
diploma
do
doesn't
dog
ejaculate
enemy
engage
ever
everybody's
everywhere
forgot
four
fuck
fucked (use fuck as a phone base)
gas
gigantic
goddamn
google
greatest
heads
hippo
hispanic
hitler
home
horrible
important
invasion
invention
japanese
java
laden
left
limited
little
machine
made
mankind's
matter
meme
memes
mexican
mexicans
mexico's
microsoft
minds
model
muslim
my
nine
osama
planet
prick
probe
programming
rectal
require
robot
role
rounder
ruby
sack
self
shiny
shovel
shut
software
sometime
sometimes
such
sudanese
ten
the
there (sounds like they're)
towards
universe
unwilling
warheads
waste
welcome
whales
will
world
+46
View File
@@ -0,0 +1,46 @@
These words aren't in the dictionary
------------------------------------
= (equals symbol)
HN
banjo-tooie
cdn
charizard
derping
dorkiness
fam
fearmongers
ganon
ganondorf
gorp
hoodie
hsv
hsv-1
hsv-2
incongruent
lel
lmao
lucario
muah
mwah
neverending
ocarina
patreon
paypal
pringles
ramen
std
strudel
sucky
there's
triforce
tweens
typescript
undressing
unicode
untrimmed
vag
wanker
wario
wristband
wwii
✓
+98
View File
@@ -0,0 +1,98 @@
; Abbreviations
; ---------------------------
AFAIK as far as i know
AFK away from keyboard
BBL be back later
BRB be right back
DT donald trump
FML fuck my life
FTL faster than light
FU fuck you
FWIW for what it's worth
GTFO get the fuck out
H8 hate
HBU how about you
HC hillary clinton
HN hacker news
IANAL i am not a lawyer
IDK i don't know
IIRC if i recall correctly
IMHO in my humble opinion
IOT internet of things
IRL in real life
LLAP live long and prosper
OMFG oh my fucking god
OMG oh my god
RT retweet
TBH to be honest
TTYL talk to you later
WRT with regard to
WTF what the fuck
WYSIWYG what you see is what you get
; Corrections
; This is a hack. They'll simply work due to the order of parsing.
; ---------------------------
911 nine one one
DAMNIT damn it
WWI world war one
WWII world war two
WWIII world war three
w/ with
; States
; Some of these are commented out as they may interfere with words
; that are more common than the state abbreviation.
; ---------------------------
; AL Alabama
; CO Colorado
; DC District of Columbia
; DE Delaware
; HI Hawaii
; ID Idaho
; IN Indiana
; LA Louisiana
; MA Massachusetts
; ME Maine
; MI Michigan
; MO Missouri
; MS Mississippi
; MT Montana
; OH Ohio
; OK Oklahoma
; OR Oregon
; PA Pennsylvania
; SD South Dakota
; ---------------------------
AK Alaska
AR Arkansas
AZ Arizona
CA California
CT Connecticut
FL Florida
GA Georgia
IA Iowa
IL Illinois
KS Kansas
KY Kentucky
MD Maryland
MN Minnesota
NC North Carolina
ND North Dakota
NE Nebraska
NH New Hampshire
NJ New Jersey
NM New Mexico
NV Nevada
NY New York
RI Rhode Island
SC South Carolina
TN Tennessee
TX Texas
UT Utah
VA Virginia
VT Vermont
WA Washington
WI Wisconsin
WV West Virginia
WY Wyoming
+33
View File
@@ -0,0 +1,33 @@
BASIS(1) B EY1 S AH0 S
BASIS B EY1 S IH0 S
BEGAN(1) B IH0 G AE1 N
BEGAN B IY0 G AE1 N
;;; "california" is more appropriate here,
;;; CA K AH1
;;; CA(1) S IY1 EY1
;;; CA(2) K AA1
DOCUMENT(1) D AA1 K Y AH0 M EH0 N T
DOCUMENT D AA1 K Y UW0 M EH0 N T
DEJA D EY2 JH AA1
FINANCIALS(1) F AY2 N AE1 N SH AH0 L Z
FINANCIALS F IH0 N AE1 N SH AH0 L Z
PAGES(1) P EY1 JH AH0 Z
PAGES P EY1 JH IH0 Z
PLACES(1) P L EY1 S AH0 Z
PLACES P L EY1 S IH0 Z
REPRESENTED(1) R EH2 P R AH0 Z EH1 N T AH0 D
REPRESENTED R EH2 P R IH0 Z EH1 N T IH0 D
RESPECT(1) R IH0 S P EH1 K T
RESPECT R IY0 S P EH1 K T
SENATE(1) S EH1 N AH0 T
SENATE S EH1 N IH0 T
SIDED(1) S AY1 D AH0 D
SIDED S AY1 D IH0 D
TONIGHT(1) T AH0 N AY1 T
TONIGHT T UW0 N AY1 T
TONIGHT'S(1) T AH0 N AY1 T S
TONIGHT'S T UW0 N AY1 T S
UNITY(0) Y UW1 N AH0 T IY0
UNITY Y UW1 N IH0 T IY0
;;; Wyoming is more appropriate,
;;; WY W EY1
+133912
View File
File diff suppressed because it is too large. Load diff
+96
View File
@@ -0,0 +1,96 @@
;;; =====================
;;; CPU Arpabet Extensions
;;; Copyright 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
;;; =====================
;;;
4CHAN F AO1 R CH AE1 N
4CHAN'S F AO1 R CH AE1 N Z
ANIME AE1 N AH0 M EY1
ANIME'S AE1 N AH0 M EY1 Z
ANIMES AE1 N AH0 M EY1 Z
API EY1 P IY1 AY1
ASSHAT AE1 S HH AE1 T
ASSWIPE AE1 S W AY0 P
BANJO-KAZOOIE B AE1 N JH OW2 K AH0 Z UW1 IY0
BATSHIT B AE1 T SH IH1 T
BIGLY B IH1 G L IY0
BITCOIN B IH1 T K OY1 N
BOOBIE B UW1 B IY0
BOOBIES B UW1 B IY0 Z
BOOBY B UW1 B IY0
BULBASAUR B AH1 L B AH0 S AO2 R
BULBASAUR'S B AH1 L B AH0 S AO2 R Z
CHARMANDER CH AA1 R M AE1 N D ER0
CHARMANDER'S CH AA1 R M AE1 N D ER0 Z
COMBINATOR K AA2 M B AH0 N EY2 T ER0
DEM D EH1 M
DEMS D EH1 M Z
DERP D ER0 P
DERPS D ER0 P S
DERPY D ER0 P IY0
DICKING D IH1 K IH0 NG
DICKWEED D IH1 K W IY2 D
DIPSHIT D IH1 P SH IH1 T
DNS D IY1 EH2 N EH1 S
DOGE D OW1 ZH
DOGECOIN D OW1 ZH K OY1 N
DOGGO D AO1 G OW2
DOGGU D AO1 G UW2
DOOP D UW1 P
DOOT D UW1 T
DORK D AO1 R K
DORKY D AO1 R K IY0
EJACULATING IH0 JH AE1 K Y UW0 L EY1 T IH0 NG
FAM F AE1 M
GIT G EH1 T
GLOMP G L AA1 M P
GLOMPING G L AA1 M P IH0 NG
HARAMBE HH AH1 R AH0 M B IY2
HARAMBE'S HH AH1 R AH0 M B IY2 Z
HOMBRES HH AA1 M B R AH0 Z
HOSTNAME HH OW1 S T N EY1 M
JIGGLYPUFF JH IH1 G L IY0 P AH1 F
JIZZ JH IH0 Z
JIZZING JH IH0 Z IH0 NG
KART K AA1 R T
KAZOOIE K AH0 Z UW1 IY0
KAZOOIE'S K AH0 Z UW1 IY0 Z
KEK K EH1 K
LOLS L AH1 L Z
LULZ L AH1 L Z
MEWTWO M Y UW1 T UW1
MILKSHAKES M IH1 L K SH EY1 K S
MILLENNIAL'S M IH0 L EH1 N IY0 AH0 L Z
MILLENNIALS M IH0 L EH1 N IY0 AH0 L Z
PHONINESS F OW1 N IY0 N AH0 S
PIKACHU P IY1 K AH0 CH UW1
POKEMON P OW1 K EY1 AH0 N
POTUS P OW1 T AH0 S
PR P IY1 AA1 R
PR'S P IY1 AA1 R Z
PRS P IY1 AA1 R Z
PUPPER P AH1 P ER0
PUPPERS P AH1 P ER0 Z
RAREWARE R EH1 R W EH2 R
REDDIT R EH1 D IH0 T
RETWEET R IH0 T W IY1 T
ROFL R AA1 F AH0 L
RSPEC AA1 R S P EH1 K
RSPECS AA1 R S P EH1 K S
SANIC S AE1 N IH0 K
SHEEPLE SH IY1 P AH0 L
SLASHDOT S L AE1 SH D AA1 T
SQUIRTLE S K W ER1 T AH0 L
SSH EH1 S EH1 S EY1 CH
SUDO S UW1 D UW1
TITTIES T IH1 T IY0 Z
TITTY T IH1 T IY0
TROGLODYTE T R AA1 G L OW2 D AY2 T
UNDELETE AH2 N D IH0 L IY1 T
UNDELETES AH2 N D IH0 L IY1 T S
UNICORNS Y UW1 N IH0 K AO2 R N Z
WAKER W EY1 K ER0
WALUIGI W AA1 L UW0 IY1 JH IY0
YCOMBINATOR W AY1 K AA2 M B AH0 N EY2 T ER0
ZOMBO Z AA1 M B OW0
ZOMBOCOM Z AA1 M B OW0 K AA2 M
+25
View File
@@ -0,0 +1,25 @@
;;; ===============================
;;; Work-Related Arpabet Extensions
;;; Copyright 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
;;; ===============================
;;;
EXPENSIFY IH0 K S P EH1 N S AH0 F AY2
GEOCODING JH IY1 OW0 K OW1 D IH0 NG
GNS JH IY1 EH1 N EH1 S
KOCHIKU K OW1 CH IY1 K UW1
MAPBOX M AE1 P B AA2 K S
MPFE EH1 M P IY1 EH1 F IY1
MULTIPASS M AH1 L T IY0 P AE2 S
MULTIPASSFE M AH1 L T IY0 P AE2 S EH1 F IY1
NAGIOS N AE1 G IY0 OW0 S
P2 P IY1 T UW1
PAGERDUTY P EY1 JH ER0 D UW1 T IY0
SQROOT S K W EH1 R R UW1 T
SQWEB S K W EH1 R W EH1 B
TRACON T R AE1 K AA1 N
;;;
;;; TODO --
;;;
;;; BITBUCKET
;;; TINMINATOR
;;; TYPESCRIPT
+10
View File
@@ -0,0 +1,10 @@
# See `ebs-deployable.sh` in the root directory to create the AWS
# EBS-deployable zip file that contains all of the artifact files.
FROM ubuntu:14.04
MAINTAINER bt <bt@brand.io>
EXPOSE 8000
# Mount files inside container.
ADD files/ ./
# Go, go, go!
CMD ./trumpet -p 8000
+8
View File
@@ -0,0 +1,8 @@
{
"AWSEBDockerrunVersion": "1",
"Ports": [
{
"ContainerPort": "8080"
}
]
}
+1
View File
@@ -0,0 +1 @@
testing bar
+1
View File
@@ -0,0 +1 @@
testing foo
+1
View File
@@ -0,0 +1 @@
typings/
+144
View File
@@ -0,0 +1,144 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
import PIXI = require('pixi.js');
import $ = require('jquery');
import { trumpAnimation } from './trump_animation';
import { getRandomInt } from './util';
/**
* The image path on AWS is stored under a content hash.
* This is injected into the meta tags before upload so we can build the path.
*/
function getImagePathPrefix() : string {
// TODO: Cache the values looked up from the DOM statically.
let environment = $('meta[name=environment]').attr('content');
switch (environment.toLowerCase()) {
case 'production':
let cdnHost = $('meta[name=cdn_host]').attr('content');
return `${cdnHost}/images`;
default:
return '/assets/images';
}
}
// TODO: Clean up this abysmal garbage.
export function setup_animation() {
let renderer = PIXI.autoDetectRenderer(800, 470);
renderer.backgroundColor = 0xffffff;
document.getElementById('animation').appendChild(renderer.view);
let stage = new PIXI.Container();
let asset: any = null;
let logo2: any = null;
function animate() {
// start the timer for the next animation loop
requestAnimationFrame(animate);
let curFrame = trumpAnimation.calculateCurrentFrame();
asset.gotoAndStop(curFrame);
// this is the main render call that makes pixi draw your container and its children.
renderer.render(stage);
}
let prefix = getImagePathPrefix();
let idleImages = [
'/trumpette/idle_0.png', // 0
'/trumpette/idle_1.png', // 1
'/trumpette/idle_2.png', // 2
'/trumpette/idle_3.png', // 3
'/trumpette/idle_4.png', // 4
];
let talkImages = [
'/trumpette/talk_0.png', // 5
'/trumpette/talk_1.png', // 6
'/trumpette/talk_2.png', // 7
'/trumpette/talk_3.png', // 8
'/trumpette/talk_4.png', // 9
'/trumpette/talk_5.png', // 10
'/trumpette/talk_6.png', // 11
'/trumpette/talk_7.png', // 12
'/trumpette/talk_8.png', // 13
];
let angryImages = [
'/trumpette/angry_0.png', // 14
'/trumpette/angry_1.png', // 15
'/trumpette/angry_2.png', // 16
'/trumpette/angry_3.png', // 17
'/trumpette/angry_4.png', // 18
'/trumpette/angry_5.png', // 19
'/trumpette/angry_6.png', // 20
'/trumpette/angry_7.png', // 21
'/trumpette/angry_8.png', // 22
'/trumpette/angry_between.png', // 23
];
let idleTextures = [];
for (let image of idleImages) {
let texture = PIXI.Texture.fromImage(`${prefix}${image}`);
idleTextures.push(texture);
}
for (let image of talkImages) {
let texture = PIXI.Texture.fromImage(`${prefix}${image}`);
idleTextures.push(texture);
}
for (let image of angryImages) {
let texture = PIXI.Texture.fromImage(`${prefix}${image}`);
idleTextures.push(texture);
}
//logo = new PIXI.Text('Jungle Horse',
// {font: '150px Impact Bold', fill: 0x000000, align: 'center'});
//let texture = PIXI.Texture.fromImage(`${prefix}/jungle_horse_lobster.png`);
let texture = PIXI.Texture.fromImage(`${prefix}/trumped.png`);
logo2 = new PIXI.Sprite(texture);
PIXI.loader.add('asset', `${prefix}/trumpette/idle_0.png`)
.load(function (loader, resources) {
// This creates a texture from a 'asset.png' image.
//asset = new PIXI.Sprite(resources.asset.texture);
asset = new PIXI.extras.MovieClip(idleTextures);
// Setup the position and scale of the asset
asset.position.x = 200;
asset.position.y = 0;
asset.scale.x = 0.5;
asset.scale.y = 0.5;
//logo.position.y = 260;
logo2.scale.x = 0.5;
logo2.scale.y = 0.5;
logo2.position.y = 160;
// Add the asset to the scene we are building.
//stage.addChild(logo);
stage.addChild(logo2);
stage.addChild(asset);
asset.play();
// kick off the animation loop (defined below)
animate();
});
blinkAnimation();
}
function blinkAnimation() {
trumpAnimation.blink();
let nextBlink = getRandomInt(1000, 4000);
setTimeout(blinkAnimation, nextBlink);
}
+37
View File
@@ -0,0 +1,37 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
/// <reference path='buzz.d.ts' />
import buzz = require('buzz');
import { trumpAnimation } from './trump_animation';
export default class Audio {
sound: buzz.Sound;
constructor() {
this.sound = null;
}
playUrl(url: string) {
this.stop();
this.sound = new buzz.sound(url);
this.sound.play();
// Possible events: 'play', 'playing'
this.sound.bind('playing', function() {
trumpAnimation.talkRandom();
});
// Possible events: 'error', 'ended', 'pause'
this.sound.bind('pause', function() {
trumpAnimation.stopTalking();
});
}
stop() {
if (this.sound != null) {
this.sound.stop();
}
}
}
+21
View File
@@ -0,0 +1,21 @@
{
"name": "jungle.horse",
"description": "none",
"main": "",
"authors": [
"Brandon Thomas <echelon@gmail.com>"
],
"license": "none",
"homepage": "https://github.com/echelon/trumpet",
"private": true,
"ignore": [
"**/.*",
"node_modules",
"bower_components",
"test",
"tests"
],
"dependencies": {
"buzz": "^1.2.0"
}
}
+18
View File
@@ -0,0 +1,18 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
declare module "buzz" {
/** CTOR */
interface SoundStatic {
new(url: any): Sound;
}
/** Sound class */
interface Sound {
bind(events: string, callback: () => void): any;
play(): any;
stop(): any;
}
export var sound: SoundStatic;
}
+108
View File
@@ -0,0 +1,108 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
/// <reference path='./typings/tsd.d.ts' />
import $ = require('jquery');
import { setup_animation } from "./animation";
import Audio from "./audio";
import { clear_url_hash, decode_url_hash, get_audio_api_url, set_url_hash } from "./url";
import { RawSentence, FilteredSentence } from "./sentence";
const INPUT = 'input#jungle';
$(function() {
(<any>window).audio = new Audio();
install_events();
initialize_from_url();
focus();
if ($('#animation').length > 0) {
// Only install animation if the page supports it.
setup_animation();
}
// TODO: For console debugging only.
(<any>window).$ = $;
});
function install_events() {
$('form').submit(function(ev: any): boolean {
let input = $('input#jungle').val(),
sentence = new RawSentence(input),
filtered = sentence.filter();
set_url_hash(filtered);
let url = get_audio_api_url(filtered);
(<any>window).audio.playUrl(url);
ev.preventDefault();
return false;
});
$('body').on('keyup', function(ev: any): any {
if (ev.keyCode === 27) {
// ESC key.
focus();
$(INPUT).val('');
(<any>window).audio.stop();
}
});
$('input#jungle').on('keyup', function(ev: any): any {
var sentence = $(INPUT).val();
if (ev.keyCode === 27) {
$(INPUT).val(''); // ESC key.
(<any>window).audio.stop();
}
});
$('button#speak').on('click', function(ev: any): any {
$('form').submit();
ev.preventDefault();
return false;
});
$('button#clear').on('click', function(ev: any): any {
$(INPUT).val(''); // ESC key.
(<any>window).audio.stop();
clear_url_hash();
ev.preventDefault();
return false;
});
$('#info').on('click', function(ev: any): any {
openInfo();
});
$('#close_top,#close_bottom').on('click', function(ev: any): any {
closeInfo();
});
}
function openInfo() {
$('aside').removeClass('hidden');
$(window).scrollTop(0);
}
function closeInfo() {
$('aside').addClass('hidden');
$(window).scrollTop(0);
}
function initialize_from_url() {
let rawSentence = decode_url_hash();
if (rawSentence == null) return;
$(INPUT)
.val(rawSentence.filter().value)
.submit();
}
function focus() {
$(INPUT).focus();
$(INPUT).select();
}
+104
View File
@@ -0,0 +1,104 @@
h1 {
font-family: ubuntu, tahoma;
font-size: 1.8em;
text-align: center;
color: #333;
}
input {
border: 1px solid #ccc;
padding: 8px;
margin-bottom: 1em;
}
input#text {
width: 70%;
}
input#vol {
width: 30px;
}
button {
background-color: #eee;
border: 1px solid #ccc;
padding: 8px;
}
select {
background-color: #eee;
border: 1px solid #ccc;
padding: 7px;
}
#control_elements {
display: inline;
}
@media (max-width: 600px) {
#control_elements {
display: block;
}
}
#main {
margin: auto;
width: 90%;
text-align: center;
}
.error input#text {
border-color: #f55;
background-color: #fee;
color: #500;
}
.ok input#text {
border-color: #5a5;
background-color: #efe;
}
#typeahead {
color: #777;
font-style: italic;
width: 100%;
height: 1em;
margin: 0 0 10px 0;
overflow: hidden;
/*text-overflow: ellipsis;*/
}
#typeahead ul {
list-style-type: none;
margin: 0;
padding: 0;
}
#typeahead li {
display: inline;
padding: 5px;
border: 1px;
}
#stats {
margin: 20px 0;
font-size: 0.8em;
color #777;
}
ul#stats {
list-style-type: none;
}
a:link, a:visited {
color: #00f;
text-decoration: none;
}
a:active, a:hover {
color: #00f;
text-decoration: underline;
}
+118
View File
@@ -0,0 +1,118 @@
// TODO: Use real AMD, and don't install crap globally.
(function() {
/** Maintain the library of vocabularies. */
window.Library = {
/** Libraries: voiceName(str)=> Vocabulary(obj) */
library: {},
/** The default speaker to choose. */
defaultSpeaker: 'trump',
/** Load the word list from the server. */
load: function(successCallback, failureCallback) {
var that = this;
$.get('/words')
.done(function(data) {
for (var speaker in data.library) {
var words = data.library[speaker];
if (words.length !== 0) {
var vocabulary = new Vocabulary(words);
that.library[speaker] = vocabulary;
}
}
if (typeof successCallback !== 'undefined') {
successCallback(that);
}
})
.fail(function() {
// Page has catastrophically failed. Suggest reload.
if (typeof successCallback !== 'undefined') {
failureCallback(that);
}
});
},
/** Return a list of speakers. */
getSpeakers: function() {
return _.keys(this.library);
},
/** Return the default speaker. */
getDefaultSpeaker: function() {
var speakers = this.getSpeakers();
if (speakers.length === 0) {
return null;
} else if (_.contains(speakers, this.defaultSpeaker)) {
return this.defaultSpeaker;
} else {
return _.first(speakers);
}
},
/** Get the `Vocabulary` for a given speaker name. */
getSpeakerVocabulary: function(speaker) {
if (!_.contains(this.getSpeakers(), speaker)) {
return null;
} else {
return this.library[speaker];
}
},
}
/**
* An individual speaker's words.
* Initialize with the subset of the JSON response corresponding to
* the speaker.
*/
var Vocabulary = function(responseSubset) {
return {
/** List of words. */
words: responseSubset.words,
/**
* Check the entirety of a sentence against words in the
* dictionary.
* Return true if okay to send to the server.
* TODO: Great candidate for unit testing.
* TODO: Handle punctuation.
*/
checkSentence: function(rawSentence) {
var splitWords = SentenceHelper.splitSentence(rawSentence);
// TODO: Bad efficiency.
for (var i in splitWords) {
if (!_.contains(this.words, splitWords[i])) {
return false;
}
}
return true;
},
/**
* Return a list of words from the dictionary based on the last
* word in the sentence. This powers type ahead.
*/
getTypeAhead: function(rawSentence) {
var splitWords = SentenceHelper.splitSentence(rawSentence),
lastWord = '';
if (splitWords.length === 0) {
return [];
}
lastWord = splitWords[splitWords.length - 1];
return this.wordsStartingWith(lastWord);
},
/** Returns a list of words starting with the string provided. */
wordsStartingWith: function(startFragment) {
// TODO: This will become inefficient with a large dictionary.
var search = (startFragment || '').toLowerCase();
if (search === '') {
return [];
} else {
return _.filter(this.words,
function(word) { return word.lastIndexOf(search, 0) === 0; });
}
},
}
}
}());
+106
View File
@@ -0,0 +1,106 @@
'use strict';
// TODO: Use real AMD, and don't install crap globally.
(function() {
/** Event callbacks */
window.Events = {
/** Handle user input inside the input box. */
handleTyping: function(ev) {
var sentence = $('input#text').val();
if (ev.keyCode === 27) {
$('input#text').val(''); // ESC key.
sentence = '';
window.waveform_player.clear();
} else if (ev.keyCode == 32) {
// Space key.
ev.stopPropagation();
}
Ui.setFromInput(sentence);
},
/** Handle typing outside of the input box. */
handleBodyTyping: function(ev) {
if (ev.target.tagName.toLowerCase() === 'input') {
return; // We'll handle this elsewhere.
}
if (ev.keyCode === 27) {
// Handle ESC key.
$('input#text').select();
Ui.clearInput();
Ui.clearSuggestedWords();
Ui.setFromInput('');
window.waveform_player.clear();
} else if (ev.keyCode == 32) {
// 'Space' key to toggle playing.
window.waveform_player.toggle();
// Prevent spacebar from causing scroll.
ev.preventDefault();
return false;
}
},
/** Handle changing voices. */
handleVoiceSelect: function(ev) {
var selectedVoice = $('select').val();
Ui.setSpeaker(selectedVoice);
},
/** Handle form submission. */
handleFormSubmit: function(ev) {
var sentence = $('#text').val(),
speaker = $('select').val(),
volume = $('input#vol').val(),
speed = $('input#speed').val(),
um = $('#use_monophones').prop('checked'),
un = $('#use_n_phones').prop('checked'),
us = $('#use_syllables').prop('checked'),
uw = $('#use_words').prop('checked'),
ue = $('#use_ends').prop('checked'),
pbp = $('input#padding_between_phones').val(),
pps = $('input#polyphone_padding_start').val(),
ppe = $('input#polyphone_padding_end').val(),
wps = $('input#word_padding_start').val(),
wpe = $('input#word_padding_end').val(),
query = encodeURIComponent(sentence),
url = '/speak?v=' + speaker
+ '&s=' + query
+ '&vol=' + volume
+ '&spd=' + speed
+ '&pbp=' + pbp
+ '&pps=' + pps
+ '&ppe=' + ppe
+ '&wps=' + wps
+ '&wpe=' + wpe
+ '&um=' + um
+ '&un=' + un
+ '&us=' + us
+ '&uw=' + uw
+ '&ue=' + ue;
ev.preventDefault();
Url.setState(speaker,
sentence,
volume,
speed,
um,
un,
us,
uw,
ue,
pbp,
pps,
ppe,
wps,
wpe);
Sound.play(url);
return false;
},
}
}());
+75
View File
@@ -0,0 +1,75 @@
'use strict';
// TODO: Use real AMD.
(function() {
/** Install event handlers. */
var installEventHandlers = function() {
$('form').submit(function(ev) { return Events.handleFormSubmit(ev); });
$('input#text').on('keyup', function(ev) { return Events.handleTyping(ev); });
$('body').on('keydown', function(ev) { return Events.handleBodyTyping(ev); });
$('select').on('change', function(ev) { return Events.handleVoiceSelect(ev); });
};
/** Init the UI, possibly from a state passed in from the URL. */
var uiInitialize = function() {
var sentence = Url.getSentence(),
speaker = Url.getSpeaker() || Library.getDefaultSpeaker(),
volume = Url.getVolume() || Volume.DEFAULT,
speed = Url.getSpeed() || 1.0,
useMonophones = Url.getUseMonophones(),
useNPhones = Url.getUseNPhones(),
useSyllables = Url.getUseSyllables(),
useWords = Url.getUseWords(),
useEnds = Url.getUseEnds(),
pbp = Url.getPaddingBetweenPhones() || 0,
pps = Url.getPolyphonePaddingStart() || 600,
ppe = Url.getPolyphonePaddingEnd() || 600,
wps = Url.getWordPaddingStart() || 600,
wpe = Url.getWordPaddingEnd() || 600,
urlPreviousState = false;
if (useMonophones === null) useMonophones = true;
if (useNPhones === null) useNPhones = true;
if (useSyllables === null) useSyllables = true;
if (useWords === null) useWords = true;
if (useEnds === null) useEnds = true;
if (speaker) {
$('select').val(speaker); // TODO: whitelist
}
if (sentence.length > 0) {
$('input#text').val(sentence);
urlPreviousState = true;
}
speed = parseFloat(speed).toFixed(2);
$('input#speed').val(speed);
$('input#vol').val(volume);
$('#use_monophones').prop('checked', useMonophones);
$('#use_n_phones').prop('checked', useNPhones);
$('#use_syllables').prop('checked', useSyllables);
$('#use_words').prop('checked', useWords);
$('#use_ends').prop('checked', useEnds);
$('input#padding_between_phones').val(pbp);
$('input#polyphone_padding_start').val(pps);
$('input#polyphone_padding_end').val(ppe);
$('input#word_padding_start').val(wps);
$('input#word_padding_end').val(wpe);
Library.load(function(library) { Ui.libraryLoadCallback(library); });
Sound.install();
if (urlPreviousState) {
$('form').submit();
}
};
$(function() {
installEventHandlers();
uiInitialize();
});
}());
+24
View File
@@ -0,0 +1,24 @@
'use strict';
// TODO: Use real AMD, and don't install crap globally.
(function() {
/**
* Static, sentence-related functions.
*/
window.SentenceHelper = {
/**
* Process a raw sentence into a cleaned up sentence (no extra
* spaces, etc.). Returns a string sentence.
*/
cleanSentence: function(rawSentence) {
return this.splitSentence(rawSentence).join(' ');
},
/** Process a raw sentence into words and other tokens. */
splitSentence: function(rawSentence) {
return _.filter(_.map(rawSentence.split(/\s+/),
function(s) { return s.toLowerCase(); }),
function(s) { return s !== ""; });
},
}
}());
+27
View File
@@ -0,0 +1,27 @@
'use strict';
// TODO: Use real AMD, and don't install crap globally.
(function() {
window.Sound = {
/** Cached sound clips. Significantly speeds up replay on mobile. */
soundCache: {},
/** Whether or not the cache is disabled. */
disableCaching: false,
/** Install the sound library. */
install: function() {
console.info('Installing sound library');
},
/** Play a sound file at a URL. */
play: function(soundUrl) {
// NB(echelon): This is the new TypeScript player wrapper.
window.waveform_player.loadAndPlay(soundUrl);
},
useCaching: function() {
return window.location.hostname != 'localhost' && !this.disableCaching;
},
};
}());
+104
View File
@@ -0,0 +1,104 @@
'use strict';
// TODO: Use real AMD.
(function() {
/** UI controls and state. */
window.Ui = {
/** The current speaker vocabulary in use. */
currentVocabulary: null,
/** Handle completion of async load of library. */
libraryLoadCallback: function(library) {
var defaultSpeaker = library.getDefaultSpeaker(),
selectedSpeaker = $('select').val(),
vocabulary = library.getSpeakerVocabulary(selectedSpeaker);
if (!selectedSpeaker) {
vocabulary = library.getSpeakerVocabulary(defaultSpeaker);
}
// TODO: Iterate over.
$('#trump_wordcount').html(library.library['trump'].words.length);
$('#hillary_wordcount').html(library.library['hillary'].words.length);
this.currentVocabulary = vocabulary;
this.setStateFromInput();
},
/** Set the voice. */
setSpeaker: function(speaker) {
var vocabulary = Library.getSpeakerVocabulary(speaker);
if (!vocabulary) {
return;
}
this.currentVocabulary = vocabulary;
this.clearInput();
this.clearSuggestedWords();
this.setStateFromInput();
Url.setState(speaker, '');
},
/** Clear the input box. */
clearInput: function() {
$('input#text').val('');
},
// TODO: Rename, redesign.
setFromInput: function(sentence) {
var suggestedWords = [];
if (this.currentVocabulary.checkSentence(sentence)) {
this.setState('ok');
} else {
this.setState('error');
}
suggestedWords = this.currentVocabulary.getTypeAhead(sentence);
this.setSuggestedWords(suggestedWords);
},
/** Set state with respect to input. */
setStateFromInput: function() {
var rawSentence = $('input#text').val();
if (this.currentVocabulary.checkSentence(rawSentence)) {
this.setState('ok');
} else {
this.setState('error');
}
},
/** Set UI state. */
setState: function(state) {
var newClass, oldClass;
if (state == 'error') {
newClass = 'error';
oldClass = 'ok';
} else if (state == 'ok') {
newClass = 'ok';
oldClass = 'error';
} else {
newClass = '';
oldClass = 'error ok';
}
$('form').removeClass(oldClass)
.addClass(newClass);
},
/** Set suggested words. Takes a list of dictionary words. */
setSuggestedWords: function(suggestedWords) {
var html = '';
suggestedWords = suggestedWords || [];
for (var i = 0; i < suggestedWords.length; i++) {
var suggestedWord = suggestedWords[i];
html += '<li>' + suggestedWord + '</li>';
}
$('#typeahead ul').html(html);
},
/** Clear the suggested words. */
clearSuggestedWords: function() {
$('#typeahead ul').html('');
},
}
}());
+280
View File
@@ -0,0 +1,280 @@
'use strict';
// TODO: Use real AMD, and don't install crap globally.
(function() {
/** Manipulate the URL. */
window.Url = {
/** JSON key for the `speaker` param. */
_SPEAKER_KEY: 'v',
/** JSON key for the `sentence` param. */
_SENTENCE_KEY: 's',
/** JSON key for the `volume` param. */
_VOLUME_KEY: 'vol',
/** JSON key for the `speed` param. */
_SPEED_KEY: 'spd',
/** JSON key for the `use_monophones` param. */
_USE_MONOPHONES_KEY: 'um',
/** JSON key for the `use_n_phones` param. */
_USE_N_PHONES_KEY: 'un',
/** JSON key for the `use_syllables` param. */
_USE_SYLLABLES_KEY: 'us',
/** JSON key for the `use_words` param. */
_USE_WORDS_KEY: 'uw',
/** JSON key for the `use_ends` param. */
_USE_ENDS_KEY: 'ue',
_PADDING_BETWEEN_PHONES: 'pbp',
_POLYPHONE_PADDING_START_KEY: 'pps',
_POLYPHONE_PADDING_END_KEY: 'ppe',
_WORD_PADDING_START_KEY: 'wps',
_WORD_PADDING_END_KEY: 'wpe',
/** Get the sentence from the `window.location`. */
getSentence: function() {
var state = this.parseState(window.location),
sentence = null;
if (!state || !(this._SENTENCE_KEY in state)) {
return '';
} else {
sentence = state[this._SENTENCE_KEY] || '';
return SentenceHelper.cleanSentence(sentence);
}
},
/** Get the speaker from the `window.location`. */
getSpeaker: function() {
var state = this.parseState(window.location);
if (!state || !(this._SPEAKER_KEY in state)) {
return null;
} else {
return state[this._SPEAKER_KEY];
}
},
/** Get the volume from the `window.location`. */
getVolume: function() {
var state = this.parseState(window.location);
if (!state || !(this._VOLUME_KEY in state)) {
return null;
} else {
return state[this._VOLUME_KEY];
}
},
/** Get the speed from the `window.location`. */
getSpeed: function() {
var state = this.parseState(window.location);
if (!state || !(this._SPEED_KEY in state)) {
return null;
} else {
return state[this._SPEED_KEY];
}
},
/** Get the "use monophones" from the `window.location`. */
getUseMonophones: function() {
var state = this.parseState(window.location);
if (!state || !(this._USE_MONOPHONES_KEY in state)) {
return null;
} else {
return state[this._USE_MONOPHONES_KEY];
}
},
/** Get the "use n-phones" from the `window.location`. */
getUseNPhones: function() {
var state = this.parseState(window.location);
if (!state || !(this._USE_N_PHONES_KEY in state)) {
return null;
} else {
return state[this._USE_N_PHONES_KEY];
}
},
/** Get the "use syllables" from the `window.location`. */
getUseSyllables: function() {
var state = this.parseState(window.location);
if (!state || !(this._USE_SYLLABLES_KEY in state)) {
return null;
} else {
return state[this._USE_SYLLABLES_KEY];
}
},
/** Get the "use words" from the `window.location`. */
getUseWords: function() {
var state = this.parseState(window.location);
if (!state || !(this._USE_WORDS_KEY in state)) {
return null;
} else {
return state[this._USE_WORDS_KEY];
}
},
/** Get the "use ends" from the `window.location`. */
getUseEnds: function() {
var state = this.parseState(window.location);
if (!state || !(this._USE_ENDS_KEY in state)) {
return null;
} else {
return state[this._USE_ENDS_KEY];
}
},
/** Get the PBP value from the `window.location`. */
getPaddingBetweenPhones: function() {
var state = this.parseState(window.location);
if (!state || !(this._PADDING_BETWEEN_PHONES in state)) {
return null;
} else {
return state[this._PADDING_BETWEEN_PHONES];
}
},
/** Get the PPS value from the `window.location`. */
getPolyphonePaddingStart: function() {
var state = this.parseState(window.location);
if (!state || !(this._POLYPHONE_PADDING_START_KEY in state)) {
return null;
} else {
return state[this._POLYPHONE_PADDING_START_KEY];
}
},
/** Get the PPE value from the `window.location`. */
getPolyphonePaddingEnd: function() {
var state = this.parseState(window.location);
if (!state || !(this._POLYPHONE_PADDING_END_KEY in state)) {
return null;
} else {
return state[this._POLYPHONE_PADDING_END_KEY];
}
},
getWordPaddingStart: function() {
var state = this.parseState(window.location);
if (!state || !(this._WORD_PADDING_START_KEY in state)) {
return null;
} else {
return state[this._WORD_PADDING_START_KEY];
}
},
getWordPaddingEnd: function() {
var state = this.parseState(window.location);
if (!state || !(this._WORD_PADDING_END_KEY in state)) {
return null;
} else {
return state[this._WORD_PADDING_END_KEY];
}
},
// TODO: Independent functions to set both speaker and sentence separately.
/** Set the `window.history` speaker, sentence, phoneme use, and word use. */
setState: function(speaker,
rawSentence,
volume,
speed,
useMonophones,
useNPhones,
useSyllables,
useWords,
useEnds,
paddingBetweenPhones,
polyphonePaddingStart,
polyphonePaddingEnd,
wordPaddingStart,
wordPaddingEnd) {
var urlHash = this.fromParams(speaker,
rawSentence,
volume,
speed,
useMonophones,
useNPhones,
useSyllables,
useWords,
useEnds,
paddingBetweenPhones,
polyphonePaddingStart,
polyphonePaddingEnd,
wordPaddingStart,
wordPaddingEnd);
console.log('set state, urlhash = ', urlHash);
window.history.replaceState(null, null, urlHash);
},
/** Encode a speaker and sentence in a URL hash. */
fromParams: function(speaker,
rawSentence,
volume,
speed,
useMonophones,
useNPhones,
useSyllables,
useWords,
useEnds,
paddingBetweenPhones,
polyphonePaddingStart,
polyphonePaddingEnd,
wordPaddingStart,
wordPaddingEnd) {
var cleanedSentence = SentenceHelper.cleanSentence(rawSentence),
cleanedSpeaker = speaker, // TODO: Check against speakers.
cleanedVolume = volume, // TODO: Filter invalid values.
state = {},
json,
uriEncoded = null;
state[this._SPEAKER_KEY] = cleanedSpeaker;
state[this._SENTENCE_KEY] = cleanedSentence;
state[this._VOLUME_KEY] = cleanedVolume;
state[this._SPEED_KEY] = speed; // TODO: Filter invalid values.
state[this._USE_MONOPHONES_KEY] = !!useMonophones;
state[this._USE_N_PHONES_KEY] = !!useNPhones;
state[this._USE_SYLLABLES_KEY] = !!useSyllables;
state[this._USE_WORDS_KEY] = !!useWords;
state[this._USE_ENDS_KEY] = !!useEnds;
state[this._PADDING_BETWEEN_PHONES] = paddingBetweenPhones;
state[this._POLYPHONE_PADDING_START_KEY] = polyphonePaddingStart;
state[this._POLYPHONE_PADDING_END_KEY] = polyphonePaddingEnd;
state[this._WORD_PADDING_START_KEY] = wordPaddingStart;
state[this._WORD_PADDING_END_KEY] = wordPaddingEnd;
json = JSON.stringify(state);
if (!json) {
return new URI('/old').hash('').toString();
} else {
uriEncoded = encodeURIComponent(json);
return new URI('/old').hash(uriEncoded).toString();
}
},
/**
* Parse the represented state (object) out of a URL hash that
* contains urlencoded JSON.
*/
parseState: function(url) {
var urlHash = new URI(url).hash(),
hashless = urlHash.replace('#', ''),
uriDecodedJson = decodeURIComponent(hashless),
state = null;
if (uriDecodedJson) {
state = JSON.parse(uriDecodedJson);
}
return state;
}
}
}());
+14
View File
@@ -0,0 +1,14 @@
'use strict';
// TODO: Use real AMD, and don't install crap globally.
(function() {
window.Volume = {
/** Default volume. */
DEFAULT: 3.0,
// TODO: Set validity in UI.
valid: function(volume) {
return !isNaN(volume);
},
}
}());
+42
View File
@@ -0,0 +1,42 @@
import { WordString } from './words';
// FIXME: no type declarations exist.
declare var buzz: any;
const URL_PREFIX = '/speak';
interface SoundUrl {
url: string;
}
// Convert a WordString into a SoundUrl.
function get_url(words: WordString) : SoundUrl {
let sentence = words.words.join(' ');
let url = URL_PREFIX + '?v=trump&vol=3&s=' + encodeURIComponent(sentence);
return { url: url };
}
// Use the buzz.js library to play a SoundUrl.
function play_url(soundUrl: SoundUrl) {
let sound = new buzz.sound(soundUrl.url);
sound.play();
}
// Play a WordString.
export function play(words: WordString) {
let soundUrl = get_url(words);
play_url(soundUrl);
}
// Play several WordStrings.
export function play_all(wordList: WordString[]) {
if (wordList.length == 0) return;
let soundUrl = get_url(wordList[0]);
let remainingWordList = wordList.slice(1, wordList.length);
let sound = new buzz.sound(soundUrl.url);
sound.bind('ended', function(e) { play_all(remainingWordList); });
sound.play();
}
+39
View File
@@ -0,0 +1,39 @@
const SENTENCES : string[] = [
"don't murder the poor people",
"hillary destroys all the good things",
"hillary is my favorite person in the world",
"i feel that war is a beautiful thing",
"i must be going crazy",
"i needed my mother and father that day",
"i remember when I was only fifty seven years of age",
"i saw children dying every way i look",
"i think your parents are absolutely stupid",
"i want all of your money immediately after this",
"i want to come in first place",
"i will always think you are stupid",
"i'm smarter than your children",
"i'm the president of reality space and the internet",
"it was a long time ago when hillary and bill got married",
"it's really amazing how dumb you are",
"just see the chaos i can create",
"let's count some numbers",
"let's make america great again",
"our civilization is better than the one they have",
"stop it",
"that man has taken all of my money",
"the total chaos and destruction is beautiful",
"there are countless children i needed to take from their countries",
"there are great images on the internet you know",
"this is the opportunity to take from the rich",
"we are all victims of harassment and abuse",
"what are you doing to those children",
"you and i are the same",
"you are all fools and i hate you",
"you fricken moron",
"that has increased my interest",
"your country is tragic and terrible",
];
export = SENTENCES;
+37
View File
@@ -0,0 +1,37 @@
import audio = require('./audio');
import words = require('./words');
import sentences = require('./sentences');
// FIXME: Cannot find import.
import $ = require('jquery');
function main() {
display_sentences();
play_sentences();
}
function display_sentences() {
let $ul = $('#sentenceList');
for (let sentence of sentences) {
$ul.append('<li>' + sentence + '</li>');
}
}
function play_sentences() {
let wordLists = [];
for (let sentence of sentences) {
let w = words.parse_words(sentence);
wordLists.push(w);
}
audio.play_all(wordLists);
}
// To export objects,
//window.words = words;
//window.audio = audio;
$(() => { main(); });
+20
View File
@@ -0,0 +1,20 @@
// Functions on words and sentences.
// TODO: SymbolString
export interface WordString {
words: string[];
}
/** Parse a raw string sentence into a Word String. */
export function parse_words(rawSentence: string) : WordString {
let words = rawSentence.split(/\s+/)
.filter(function(w) { return w !== "" })
.map(function(w) { return w.toLowerCase(); });
return { words: words };
}
/*export = {
parse_words: parse_words
};*/
+16
View File
@@ -0,0 +1,16 @@
{
"dependencies": {
"bower": "^1.7.9",
"buzz": "^1.2.0",
"jquery": "^3.0.0",
"pixi.js": "^4.1.0",
"ts-loader": "^0.8.2",
"typescript": "^1.8.10",
"urijs": "^1.18.2",
"vue": "^1.0.28",
"wavesurfer.js": "^1.1.12"
},
"devDependencies": {
"webpack": "^1.13.3"
}
}
+31
View File
@@ -0,0 +1,31 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
export class RawSentence {
value: string;
constructor(sentence: string) {
this.value = sentence;
}
/**
* Process a raw sentence into a cleaned up sentence (no extra
* spaces, etc.) Returns a string sentence.
*/
filter(): FilteredSentence {
let lower = this.value.toLowerCase(),
split = lower.split(/\s+/),
filtered = split.filter(function(s: string): boolean { return s !== ''; }),
joined = filtered.join(' ');
return new FilteredSentence(joined);
}
}
export class FilteredSentence {
value: string;
constructor(sentence: string) {
this.value = sentence;
}
}
+275
View File
@@ -0,0 +1,275 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
enum TrumpState {
Idle,
Blinking,
Talking,
TalkingStop,
AngryTalkingStart,
AngryTalkingLoop,
AngryTalkingStop,
}
/**
* State machine that controls which animation state Trump is in, and
* frame ticker to calculate the current frame at any given moment.
*/
class TrumpAnimation {
private currentState: TrumpState;
private currentFrame: number;
private clock: number;
constructor() {
this.currentState = TrumpState.Idle;
this.currentFrame = 0;
}
/**
* Set Trump in the "blink" animation state.
*/
blink() {
if (this.currentState == TrumpState.Idle) {
this.currentState = TrumpState.Blinking;
}
}
/**
* Set Trump in the "talk" animation state.
*/
talk() {
this.currentState = TrumpState.Talking;
}
/**
* Set Trump in the "angry talk" animation state.
*/
angryTalk() {
this.currentState = TrumpState.AngryTalkingStart;
}
/**
* Use a random talking animation.
*/
talkRandom() {
if (Math.random() >= 0.5) {
this.talk();
} else {
this.angryTalk();
}
}
/**
* Stop trump from talking.
*/
stopTalking() {
switch(this.currentState) {
case TrumpState.Talking:
this.currentState = TrumpState.TalkingStop;
break;
case TrumpState.AngryTalkingStart:
case TrumpState.AngryTalkingLoop:
this.currentState = TrumpState.AngryTalkingStop;
break;
default:
break;
}
}
/**
* Get the current frame index.
*/
calculateCurrentFrame(): number {
switch (this.currentState) {
case TrumpState.Idle:
this.currentFrame = 0;
break;
case TrumpState.Blinking:
this.calculateBlinking();
break;
case TrumpState.Talking:
this.calculateTalking();
break;
case TrumpState.TalkingStop:
this.calculateTalkingStop();
break;
case TrumpState.AngryTalkingStart:
this.calculateAngryTalkingStart();
break;
case TrumpState.AngryTalkingLoop:
this.calculateAngryTalkingLoop();
break;
case TrumpState.AngryTalkingStop:
this.calculateAngryTalkingStop();
break;
default:
break;
}
return this.currentFrame;
}
private calculateBlinking() {
if (this.clock == null) {
this.clock = window.performance.now();
this.currentFrame = 0;
} else {
let delta = window.performance.now() - this.clock;
if (delta > 200) {
this.currentFrame = 0;
this.clock = null;
this.currentState = TrumpState.Idle;
} else if (delta > 150) {
this.currentFrame = 4;
} else if (delta > 60) {
this.currentFrame = 3;
} else if (delta > 20) {
this.currentFrame = 2;
} else if (delta > 10) {
this.currentFrame = 1;
}
}
}
private calculateTalking() {
if (this.clock == null) {
this.clock = window.performance.now();
this.currentFrame = 5;
} else {
let delta = window.performance.now() - this.clock;
if (delta > 550) {
this.currentFrame = 5;
this.clock = null;
} else if (delta > 580) {
this.currentFrame = 6;
} else if (delta > 540) {
this.currentFrame = 7;
} else if (delta > 500) {
this.currentFrame = 8;
} else if (delta > 460) {
this.currentFrame = 9;
} else if (delta > 420) {
this.currentFrame = 10;
} else if (delta > 400) {
this.currentFrame = 11;
} else if (delta > 360) {
this.currentFrame = 12;
} else if (delta > 320) {
this.currentFrame = 13;
} else if (delta > 280) {
this.currentFrame = 12;
} else if (delta > 240) {
this.currentFrame = 11;
} else if (delta > 200) {
this.currentFrame = 10;
} else if (delta > 160) {
this.currentFrame = 9;
} else if (delta > 120) {
this.currentFrame = 8;
} else if (delta > 80) {
this.currentFrame = 7;
} else if (delta > 40) {
this.currentFrame = 6;
}
}
}
private calculateTalkingStop() {
if (this.currentFrame > 13 || this.currentFrame < 5) {
this.currentState = TrumpState.Idle;
this.currentFrame = 0;
} else {
if (this.clock == null) {
this.clock = window.performance.now();
}
let delta = window.performance.now() - this.clock;
if (delta > 40) {
this.currentFrame -= 1;
this.clock = window.performance.now();
}
}
}
private calculateAngryTalkingStart() {
const frame_rate = 50;
if (this.clock == null) {
this.clock = window.performance.now();
this.currentFrame = 23;
} else {
let delta = window.performance.now() - this.clock;
if (delta > frame_rate * 6) {
this.currentFrame = 19;
this.clock = null;
this.currentState = TrumpState.AngryTalkingLoop;
} else if (delta > frame_rate * 5) {
this.currentFrame = 18;
} else if (delta > frame_rate * 4) {
this.currentFrame = 17;
} else if (delta > frame_rate * 3) {
this.currentFrame = 16;
} else if (delta > frame_rate * 2) {
this.currentFrame = 15;
} else if (delta > frame_rate * 1) {
this.currentFrame = 14;
}
}
}
private calculateAngryTalkingLoop() {
const frame_rate = 80;
if (this.clock == null) {
this.clock = window.performance.now();
this.currentFrame = 19;
} else {
let delta = window.performance.now() - this.clock;
if (delta > frame_rate * 6) {
this.currentFrame = 19;
this.clock = window.performance.now();
} else if (delta > frame_rate * 5) {
this.currentFrame = 20;
} else if (delta > frame_rate * 4) {
this.currentFrame = 21;
} else if (delta > frame_rate * 3) {
this.currentFrame = 22;
} else if (delta > frame_rate * 2) {
this.currentFrame = 21;
} else if (delta > frame_rate * 1) {
this.currentFrame = 20;
}
}
}
private calculateAngryTalkingStop() {
const frame_rate = 50;
if ((this.currentFrame > 22 || this.currentFrame < 14) && this.currentFrame != 23) {
// TODO FIXME - nasty code
this.currentFrame = 23;
this.clock = null;
} else if (this.currentFrame == 23) {
if (this.clock == null) {
this.clock = window.performance.now();
} else {
let delta = window.performance.now() - this.clock;
if (delta > frame_rate) {
this.currentFrame = 0;
this.clock = null;
this.currentState = TrumpState.Idle;
}
}
} else {
if (this.clock == null) {
this.clock = window.performance.now();
}
let delta = window.performance.now() - this.clock;
if (delta > frame_rate) {
this.currentFrame -= 1;
this.clock = window.performance.now();
}
}
}
}
export const trumpAnimation = new TrumpAnimation();
// TODO TEMP
(<any>window).trumpAnimation = trumpAnimation;
+12
View File
@@ -0,0 +1,12 @@
{
"filesGlob": [
"./**/*.ts",
"!./node_modules/**/*.ts"
],
"files": [
"./main.ts",
"./typings/pixi.js/pixi.js.d.ts",
"./typings/tsd.d.ts",
"./typings/vue/vue.d.ts"
]
}
+21
View File
@@ -0,0 +1,21 @@
{
"version": "v4",
"repo": "borisyankov/DefinitelyTyped",
"ref": "master",
"path": "typings",
"bundle": "typings/tsd.d.ts",
"installed": {
"jquery/jquery.d.ts": {
"commit": "44be13ae844ec0721109542d5902e41438d72855"
},
"vue/vue.d.ts": {
"commit": "3cc599268d2dabe3c3ee194e92965ee87440e9c3"
},
"pixi.js/pixi.js.d.ts": {
"commit": "3cc599268d2dabe3c3ee194e92965ee87440e9c3"
},
"urijs/URIjs.d.ts": {
"commit": "44be13ae844ec0721109542d5902e41438d72855"
}
}
}
+5
View File
@@ -0,0 +1,5 @@
{
"globalDependencies": {
"vue": "registry:dt/vue#1.0.21+20160929090511"
}
}
+76
View File
@@ -0,0 +1,76 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
import $ = require('jquery');
import { RawSentence, FilteredSentence } from "./sentence";
import URI = require('urijs');
/**
* The API server hostname (including HTTP scheme).
* This is injected into the meta tags before upload so we can build the path.
*/
function getApiHost(): string {
let environment = $('meta[name=environment]').attr('content');
switch (environment.toLowerCase()) {
case 'production':
return $('meta[name=api_host]').attr('content');
default:
return '';
}
}
/**
* Sentence -> Audio generation API URL
*/
export function get_audio_api_url(sentence: FilteredSentence) : string {
// I may expand the API later, but for now this is all I'll expose.
let apiHost = getApiHost(),
encoded = encodeURIComponent(sentence.value);
return `${apiHost}/speak?v=trump&vol=3&s=${encoded}`;
}
/**
* Reset the URL.
*/
export function clear_url_hash() {
window.history.replaceState(null, null, '/');
}
/**
* Sentence -> Persistent URL hash
*/
export function set_url_hash(sentence: FilteredSentence) {
let state = { 's': sentence.value },
json = JSON.stringify(state),
urlHash = '';
if (!json) {
urlHash = new URI('').hash('').toString();
} else {
let uriEncoded = encodeURIComponent(json);
urlHash = new URI('').hash(uriEncoded).toString();
}
window.history.replaceState(null, null, urlHash);
}
/**
* Persistent URL hash -> Raw Sentence
*/
export function decode_url_hash(): RawSentence {
let urlHash = new URI(window.location).hash(),
hashless = urlHash.replace('#', ''),
uriDecodedJson = decodeURIComponent(hashless),
state = null;
if (uriDecodedJson) {
state = JSON.parse(uriDecodedJson);
}
if (!state || !('s' in state)) {
return null;
} else {
return new RawSentence(state['s']);
}
}
+7
View File
@@ -0,0 +1,7 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
export function getRandomInt(min: number, max: number) : number {
min = Math.ceil(min);
max = Math.floor(max);
return Math.floor(Math.random() * (max - min)) + min;
}
+88
View File
@@ -0,0 +1,88 @@
// FIXME: no type declarations exist.
declare var $: any;
declare var WaveSurfer: any;
type WaveSurfer = any;
export default class WaveformPlayer {
/** Sound and Vizualization API. */
wavesurfer: WaveSurfer;
/**
* Whether the player is "cleared" of an audio file.
* Necessary to maintain our own state since the library will happily
* resume playing if told, even after it is "cleared".
*/
isCleared: boolean;
constructor(cssSelector: string, waveColor?: string,
progressColor?: string) {
waveColor = waveColor
|| $('meta[name=wave_color]').attr('content')
|| 'teal';
progressColor = progressColor
|| $('meta[name=progress_color]').attr('content')
|| 'purple';
this.wavesurfer = WaveSurfer.create({
container: cssSelector,
waveColor: waveColor,
progressColor: progressColor,
});
this.isCleared = true;
}
/** Load the wav url. */
load(wavUrl: string) {
this.wavesurfer.load(wavUrl);
this.isCleared = false;
}
/** Load the wav url and immediately play it. */
loadAndPlay(wavUrl: string) {
let that = this;
this.load(wavUrl);
this.isCleared = false;
this.wavesurfer.on('ready', function () {
that.wavesurfer.play();
});
}
/** In place pause. */
pause() {
this.wavesurfer.pause();
}
/** Resume playing at current position. */
play() {
if (this.isCleared) {
return; // Nothing to play.
}
this.wavesurfer.play();
}
/** Toggle play/pause states. */
toggle() {
if (this.wavesurfer.isPlaying()) {
this.pause();
} else {
this.play();
}
}
/** Stop and return to the beginning. */
stop() {
this.wavesurfer.stop();
}
/** Stop and clear the waveform data. */
clear() {
this.wavesurfer.empty();
this.isCleared = true;
}
}
window['waveform_player'] = new WaveformPlayer('#waveform');
+36
View File
@@ -0,0 +1,36 @@
var webpack = require('webpack');
var plugins = [];
var minify = JSON.parse(process.env.MINIFY || '0');
if (minify) {
plugins.push(new webpack.optimize.UglifyJsPlugin({
compress: { warnings: false }
}));
}
module.exports = {
entry: {
// Frontend code
main: './main.ts',
// Old code
waveform_component: './waveform_component.ts',
//testing_app: './old_test_page/testing_app.ts', // TODO: Rename broke this
},
resolve: {
extensions: ['', '.webpack.js', '.web.js', '.ts', '.js']
},
output: {
path: '../web/output',
filename: '[name].built.js'
},
module: {
loaders: [
{
loader: 'ts-loader',
test: /\.ts$/
}
]
},
plugins: plugins,
}
+56
View File
@@ -0,0 +1,56 @@
'use strict';
var gulp = require('gulp'),
concat = require('gulp-concat'),
filter = require('gulp-filter'),
mainBowerFiles = require('main-bower-files'),
rename = require('gulp-rename'),
uglify = require('gulp-uglify'),
watch = require('gulp-watch');
var buildLibraries = function() {
// See https://gist.github.com/ktmud/9384509
return gulp.src(mainBowerFiles())
.pipe(filter('**/*.js')) // filter out CSS (bootstrap, etc.)
.pipe(uglify())
.pipe(concat('libraries.js'))
.pipe(gulp.dest('web/output/'));
}
var buildCss = function() {
return gulp.src('frontend/old_frontend/css/*.css')
.pipe(concat('style.css'))
.pipe(gulp.dest('web/output/'));
}
var buildJs = function() {
return gulp.src([
'frontend/old_frontend/js/sentence.js',
'frontend/old_frontend/js/dictionary.js',
'frontend/old_frontend/js/sound.js',
'frontend/old_frontend/js/volume.js',
'frontend/old_frontend/js/url.js',
'frontend/old_frontend/js/ui.js',
'frontend/old_frontend/js/events.js',
'frontend/old_frontend/js/initialize.js',
])
.pipe(uglify())
.pipe(concat('script.js'))
.pipe(gulp.dest('web/output/'));
}
gulp.task('bower', buildLibraries);
gulp.task('js', buildJs);
gulp.task('css', buildCss);
gulp.task('watch', function() {
gulp.watch(['bower.json', '.bowerrc'], ['bower']);
watch([ 'frontend/old_frontend/js/*.js' ], buildJs);
watch([ 'frontend/old_frontend/css/*.css' ], buildCss);
});
gulp.task('build-development', ['bower', 'js', 'css']);
gulp.task('build-production', ['bower', 'js', 'css', 'compress']); // TODO: Compress
gulp.task('default', ['build-development']);
+25
View File
@@ -0,0 +1,25 @@
#!/bin/bash
# Upload everything to the server.
declare -r hostname="colossus960.startdedicated.com"
# Sound Library
pushd ~/Dropbox/jungle.horse
rsync -utrv sounds bt@$hostname:/home/bt/dev/trumpet
popd
# Config and Dictionaries
rsync -utrv {dictionary,config.toml} bt@$hostname:/home/bt/dev/trumpet
# Frontend
rsync -utrv web/{images,*css,*html,*txt} bt@$hostname:/home/bt/dev/trumpet/web
pushd frontend
MINIFY=1 webpack
popd
gulp
rsync -utrv web/output bt@$hostname:/home/bt/dev/trumpet/web
# Server
cargo build --release
rsync -utrv target/release/trumpet bt@$hostname:/home/bt/dev/trumpet
+19
View File
@@ -0,0 +1,19 @@
{
"devDependencies": {
"bower": "^1.7.2",
"gulp": "^3.9.0",
"gulp-bower": "0.0.11",
"gulp-bower-files": "^0.2.7",
"gulp-concat": "^2.6.0",
"gulp-filter": "^3.0.1",
"gulp-rename": "^1.2.2",
"gulp-uglify": "^1.5.1",
"gulp-watch": "^4.3.5",
"main-bower-files": "^2.11.1"
},
"dependencies": {
"jquery": "^3.0.0",
"ts-loader": "^0.8.2",
"typescript": "^1.8.10"
}
}
+45
View File
@@ -0,0 +1,45 @@
# bt(note): found at https://www.youtube.com/watch?v=f7ZPc3OR9EU
# segment_files.praat
# Ricardo A. H. Bion, 02/05/2009
form SEGMENT FILES
sentence File_to_read_from: /home/bt/dev/trumpet/samples/sample-frankly-ukraine.wav
sentence Directory_to_save_segmented_files: /home/bt/dev/trumpet/output
endform
writeInfoLine: "test"
select all
nocheck Remove
appendInfoLine: "reading"
sound = Read from file... 'file_to_read_from$'
text = To TextGrid (silences)... 100 0 -40 0.4 0.1 silent sounding
select all
Extract intervals where... 1 no "is equal to" sounding
select text
plus sound
Remove
appendInfoLine: "read"
select all
numberOfSounds = numberOfSelected()
for i to numberOfSounds
appendInfoLine: "testx"
sound_'i' = selected("Sound", 'i')
endfor
for i to numberOfSounds
appendInfoLine: "testy"
select sound_'i'
#Write to AIFF file... 'directory_to_save_segmented_files$''i'.aiff
Write to WAV file... 'directory_to_save_segmented_files$''i'.wav
endfor
appendInfoLine: "done"
select all
Remove
+238
View File
@@ -0,0 +1,238 @@
# This script creates a TextGrid object for a LongSound object and sets boundaries at pauses
# on the basis of an intensity analysis.
# The boundaries will be set either in the centre time of a pause or at the beginning
# and end of pauses. In the latter case you can also give a time margin that will be left
# around the sound segments. Use a bigger margin if the pause detection
# does not seem to work accurately. Different amounts of background noise can change the
# ideal pause detection parameters, and different speakers have different pause duration,
# so you should also try to modify the pause detection parameters to improve the accuracy.
# How to run this script:
# 1. Open the Praat program
# 2. Before running the script, you should open a LongSound file (Read menu)
# and make sure it is selected in the Object window.
# 3. Choose Open script from the Control menu
# 4. Look for this text file.
# 5. Then choose Run from the Run menu of the script window.
#
# This script is distributed under the GNU General Public License.
# Copyright Mietta Lennes 25.1.2002
#
# Fixes:
# - Added checks for existing boundaries and overlapping pause regions. ML 23.1.2006
#******************************************************************************************************
# DEFAULT VALUES (initialization of variables)
# The name of the selected LongSound object is put to string soundname$:
form Give the parameters for pause analysis
comment This script marks the pauses in the LongSound to the IntervalTier of the TextGrid.
comment Give the time period you wish to include (The TextGrid will be overwritten!):
real Starting_time_(seconds) 0
real Finishing_time_(0=all) 0
comment The following criteria define a pause:
positive Minimum_duration_(seconds) 0.6
positive Maximum_intensity_(dB) 59
comment Give the intensity analysis parameters:
positive Minimum_pitch_(Hz) 100
integer Time_step_(0=auto) 0
comment Give the window size for the intensity analyses (smaller window requires less memory):
positive Window_size_(seconds) 20
choice Boundary_placement 2
button One boundary at the center of each pause
button Two boundaries with a time margin of:
positive Margin_(seconds) 0.1
comment (The margin will not be used if the pause is shorter than 2 * margin.)
boolean Mark_pause_intervals_with_xxx 0
comment Save TextGrid file to folder:
text folder /home/lennes/
comment The script will pause after calculating 4 windows, so you can interrupt the script and check if the pause detection works optimally.
endform
soundname$ = selected$ ("LongSound")
To TextGrid... sentence
if fileReadable ("'folder$''soundname$'.TextGrid")
pause The file 'folder$''soundname$'.TextGrid already exists. Do you want to overwrite it?
endif
select TextGrid 'soundname$'
endofsound = Get finishing time
select LongSound 'soundname$'
pausenumber = 0
duration = 0
count = 0
loops = 0
pauses_found = 0
windowstart = 0
windowend = 0
frame = 0
frames = 0
time = 0
intensity = 0
pausedetected = 0
pausestart = 0
pauseend = 0
pausenumber = 0
halfpause = 0
# This form prompts for parameters for the pause analysis:
if finishing_time < 0
exit Finishing time must be greater than or equal to zero! (If you give a zero, the whole LongSound will be analysed.)
endif
if finishing_time = 0
finishing_time = endofsound
endif
#******************************************************************************************************
# BEGIN
#******************************************************************************************************
# DIVIDE LONGSOUND INTO SHORTER PERIODS AND LOOP THROUGH EACH
duration = finishing_time - starting_time
#--------------------------------------------------------------------------------------------------
# Default number of loops is 1
loops = 1
# but if the period to be analysed is longer than 60 seconds, it will be divided into 60-second
# periods for which the analysis is made:
if duration > window_size
loops = ceiling ((duration/window_size))
endif
#--------------------------------------------------------------------------------------------------
# START LOOPING THROUGH SHORT WINDOWS HERE
count = 1
latest_endboundary = 0
while count <= loops
if count = 5
pause Continue?
endif
# Create a window of the LongSound and extract it for analysis
windowstart = starting_time + ((count - 1) * window_size)
windowend = starting_time + (count * window_size)
if windowend > endofsound
windowend = endofsound
endif
if windowend > finishing_time
windowend = finishing_time
endif
select LongSound 'soundname$'
Extract part... windowstart windowend yes
windowname$ = "Window_" + "'count'" + "_of_" + "'loops'"
echo Analysing Intensity window 'count' of 'loops'
if count < 5
printline The script will pause after calculating 4 windows, so you can check the result...
endif
Rename... 'windowname$'
#--------------------------------------------------------------------------------------------------
# CALCULATE INTENSITY
To Intensity... minimum_pitch time_step
frames = Get number of frames
#--------------------------------------------------------------------------------------------------
# Check the pause criteria
pauseend = 0
frame = 1
#--------------------------------------------------------------------------------------------------
# Loop through all frames in the Intensity object:
while frame <= frames
select Intensity 'windowname$'
intensity = Get value in frame... frame
time = Get time from frame... frame
if intensity > maximum_intensity
# If the end of an earlier detected possible pause has been reached:
if pausedetected = 1
if frame - 1 < 1
pauseend = windowstart
else
pauseend = Get time from frame... (frame - 1)
endif
pausedetected = 0
endif
# If below intensity limit, a possible new pause is started if one hasn't been detected yet:
elsif pausedetected = 0
pausestart = Get time from frame... frame
pauseend = 0
pausedetected = 1
pausenumber = pausenumber + 1
# If a detected pause just continues, do nothing special.
endif
#--------------------------------------------------------------------------------------------------
# IF PAUSE CRITERIA ARE FULFILLED, ADD A BOUNDARY OR TWO TO TEXTGRID
if pauseend > 0
pauseduration = pauseend - pausestart
if pauseduration >= minimum_duration
select TextGrid 'soundname$'
halfpause = pauseduration / 2
if boundary_placement = 1
boundary = pausestart + halfpause
call BoundaryCheck
if boundaryexists = 0
Insert boundary... 1 boundary
latest_endboundary = boundary
endif
else
boundary = 0
if pauseduration >= (2 * margin)
if pausestart > margin
boundary = pausestart + margin
call BoundaryCheck
if boundaryexists = 0 and boundary > latest_endboundary
Insert boundary... 1 boundary
endif
#If the pause overlaps with the preceding pause, do a merge:
if boundary = latest_endboundary
Remove boundary at time... 1 boundary
endif
endif
if mark_pause_intervals_with_xxx = 1
pauseinterval = Get interval at time... 1 boundary
Set interval text... 1 pauseinterval xxx
endif
boundary = pauseend - margin
call BoundaryCheck
if boundaryexists = 0 and boundary > latest_endboundary
Insert boundary... 1 boundary
latest_endboundary = boundary
endif
else
if pauseend < (endofsound - margin)
boundary = pausestart + halfpause
call BoundaryCheck
if boundaryexists = 0 and boundary > latest_endboundary
Insert boundary... 1 boundary
latest_endboundary = boundary
endif
endif
endif
endif
pauseend = 0
pauses_found = pauses_found + 1
Write to text file... 'folder$''soundname$'.TextGrid
endif
endif
frame = frame + 1
# When all frames in the intensity analysis have been looked at, end the frame loop.
endwhile
#--------------------------------------------------------------------------------------------------
select Sound 'windowname$'
Remove
select Intensity 'windowname$'
Remove
# END LOOPING THROUGH WINDOWS HERE
count = count + 1
endwhile
select TextGrid 'soundname$'
Write to text file... 'folder$''soundname$'.TextGrid
echo Ready! The TextGrid file was saved as 'folder$''soundname$'.TextGrid.
#******************************************************************************************************
procedure BoundaryCheck
# This procedure checks whether a boundary already exists at a given time (in tier 1).
# Added 23.1.2006
tmpint = Get interval at time... 1 boundary
tmpstart = Get starting point... 1 tmpint
if tmpstart <> boundary
boundaryexists = 0
else
boundaryexists = 1
endif
endproc
+135
View File
@@ -0,0 +1,135 @@
#########################################################################################################
### save-small-files.praat
###
### Saves little sound files from one big sound file.
###
### Opens a soundfile as a LongSound object and reads in its associated TextGrid file.
###
### Extracts the portion of the sound file corresponding to all non-empty intervals on a specified tier,
### with an optional buffer on each end.
###
### Saves each extracted portion to a .wav file named:
### (<prefix>)<interval label>_<time_value>(<suffix>).wav
###
### NB: The time value (in seconds to two decimal places) of the beginning of the segment
### in the original sound file is included in the file name.
###
### This disambiguates between two tokens of the same word and also allows one to find the word
### in the longer sound file, if need be.
###
### Creates and saves a (optionally labelled) TextGrid file for each word.
###
### Creates a text file named:
### <orginal long sound file name>_list.txt
### that contains a list of all the .wav files created. Useful for keeping track of what came from where.
###
###
### Praat 4.3.29
###
### Pauline Welby welby@icp.inpg.fr
### October 28, 2004
###
#########################################################################################################
# form that asks user for the directories containing file to be worked on,
# and for other information
form Input directory name with final slash
comment Enter parent directory where sound file is kept:
sentence soundDir /home/bt/dev/trumpet/samples
comment Enter directory where TextGrid file is kept:
sentence textDir /home/bt/dev/trumpet/samples
comment Enter directory to which created sound files should be saved:
sentence outDir /home/bt/dev/trumpet/samples
comment Specify tier name:
sentence tierName words
comment Specify length of left and right buffer (in seconds):
positive margin 0.0000005
comment Optional prefix:
sentence prefix
comment Optional suffix (.wav will be added anyway):
sentence suffix
comment Append time point?
boolean append_time no
comment Enter basename of soundfile (without .wav extension)
sentence baseFile saakuru
endform
# delete any existing record file
filedelete 'outDir$'\'baseFile$'_list.txt
numberOfFiles = 1
for ifile to numberOfFiles
# Read in the Sound and TextGrid files
Read from file... C:\Users\idarcy\Desktop\Praat_Data\Chisato\saakuru.TextGrid
Read from file... 'textDir$'\'baseFile$'.TextGrid
Open long sound file... 'soundDir$'\'baseFile$'.wav
# Go through tiers and extract info
select TextGrid 'baseFile$'
nTiers = Get number of tiers
for i from 1 to 'nTiers'
tname$ = Get tier name... 'i'
if tname$ = "'tierName$'"
# Find non-empty intervals
nInterv = Get number of intervals... 'i'
for j from 1 to 'nInterv'
lab$ = Get label of interval... 'i' 'j'
if lab$ != ""
# Get time values for start and end of the interval
begwd = Get starting point... 'i' 'j'
endwd = Get end point... 'i' 'j'
# Add buffers, if specified
begfile = 'begwd'-'margin'
endfile = 'endwd'+'margin'
# Create and save small .wav file
select LongSound 'baseFile$'
Extract part... 'begfile' 'endfile' yes
if append_time = 1
Write to WAV file... 'outDir$'\'prefix$''lab$'-'begwd:2''suffix$'.wav
else
Write to WAV file... 'outDir$'\'prefix$''lab$''suffix$'.wav
endif
# Write label of each saved interval to a text file (keeps a record of origin of small soundfiles)
fileappend 'outDir$'\'baseFile$'_list.txt 'prefix$''lab$'_'begwd:1''suffix$'.wav 'newline$'
## Object cleanup
select Sound 'baseFile$'
Remove
## Re-select TextGrid
select TextGrid 'baseFile$'
endif
endfor
endfor
# Complete object cleanup
select TextGrid 'baseFile$'
plus LongSound 'baseFile$'
Remove
select TextGrid 'baseFile$'
Remove
####### END OF SCRIPT #######
+3
View File
@@ -0,0 +1,3 @@
# Dummy Python pip requirements.txt
# This is to satisfy the `fabistrano` deploy script expectations.
# TODO: Remove requirement for requirements.txt
+22
View File
@@ -0,0 +1,22 @@
{
"Version": "2012-10-17",
"Id": "Policy1478413965163",
"Statement": [
{
"Sid": "Stmt1478413958263",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::961528074194:user/junglehorse"
},
"Action": "s3:*",
"Resource": "arn:aws:s3:::junglehorse-frontend/*"
},
{
"Sid": "PublicReadGetObject",
"Effect": "Allow",
"Principal": "*",
"Action": ["s3:GetObject"],
"Resource": "arn:aws:s3:::junglehorse-frontend/*"
}
]
}
+13
View File
@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<CORSRule>
<AllowedOrigin>*.jungle.horse</AllowedOrigin>
<AllowedOrigin>*.junglehorse.com</AllowedOrigin>
<AllowedOrigin>jungle.horse</AllowedOrigin>
<AllowedOrigin>junglehorse.com</AllowedOrigin>
<AllowedMethod>GET</AllowedMethod>
<AllowedMethod>HEAD</AllowedMethod>
<MaxAgeSeconds>3000</MaxAgeSeconds>
<AllowedHeader>*</AllowedHeader>
</CORSRule>
</CORSConfiguration>
+85
View File
@@ -0,0 +1,85 @@
#!/bin/bash
# Upload all the things to S3.
set -ex
build_assets() {
# TODO: For now, manually invoke this and then copy `./web/output` into `./brandon.audio/output`
#gulp # FIXME(2019-01-30): Not sure why this isn't working. Not sure if critical.
pushd ./frontend
MINIFY=1 webpack
popd
}
# Upload the assets that undergo frequent change (JS, CSS, etc.)
# Don't include large, infrequently changing files here as it's expensive.
upload_assets() {
echo "> Build artifact."
build_assets
# Calculate output directory.
echo "> Calculate artifact SHA."
pushd ./brandon.audio/output > /dev/null
declare checksum=$(find . -type f -exec md5sum {} + | awk '{print $1}' | \
sort | md5sum | awk '{print $1}')
popd > /dev/null
declare -r output_dir="deploy/${checksum}"
# Prep upload artifact.
echo "> Prep upload of artifact ${checksum}."
pushd ./brandon.audio
mkdir -p $output_dir
cp *css $output_dir
cp *html $output_dir
#cp output/* $output_dir # FIXME(2019-01-30): Not sure why this isn't working. Needed?
# Modify URLs in index.html
echo "> Modify asset paths, etc. in index.html."
pushd $output_dir
# /assets/output/main.built.js -> /assets/main.built.js
sed -i "s/assets\/output/assets/g" index.html
# /assets/main.built.js -> //cdn.brandon.audio/assets/${checksum}/main.built.js
sed -i "s/\/assets/\/\/cdn.brandon.audio\/assets\/${checksum}/g" index.html
# Variable substitutions
sed -i "s/asset_content_hash/${checksum}/g" index.html
sed -i "s/DEVELOPMENT/production/g" index.html
sed -i "s/API_HOST/http:\/\/api.brandon.audio/g" index.html
sed -i "s/CDN_HOST/\/\/cdn.brandon.audio/g" index.html
popd
aws s3 cp $output_dir s3://cdn.brandon.audio/assets/${checksum} --recursive
aws s3 cp $output_dir/index.html s3://brandon.audio/index.html
#aws s3 cp $output_dir/index.html s3://junglehorse.com/index.html
popd
}
# Upload files to S3 that infrequently change.
# Don't run this often as it adds to PutObject costs!
upload_stable_assets() {
echo '> Upload stable assets (images, robots.txt, etc.)'
pushd web
# S3 Bucket
# brandon.audio website
aws s3 cp error.html s3://brandon.audio/
aws s3 cp favicon.ico s3://brandon.audio/favicon.ico
aws s3 cp robots.txt s3://brandon.audio/
# S3 Bucket
# junglehorse.com website
#aws s3 cp error.html s3://junglehorse.com/
#aws s3 cp favicon.ico s3://junglehorse.com/favicon.ico
#aws s3 cp robots.txt s3://junglehorse.com/
# Cloudfront bucket.
# Keep error.html in both buckets.
aws s3 cp error.html s3://cdn.brandon.audio/
aws s3 cp images/ s3://cdn.brandon.audio/images/ --recursive
popd
}
upload_assets
upload_stable_assets
Executable
+84
View File
@@ -0,0 +1,84 @@
#!/bin/bash
# Upload all the things to S3.
set -ex
build_assets() {
gulp
pushd ./frontend
MINIFY=1 webpack
popd
}
# Upload the assets that undergo frequent change (JS, CSS, etc.)
# Don't include large, infrequently changing files here as it's expensive.
upload_assets() {
echo "> Build artifact."
build_assets
# Calculate output directory.
echo "> Calculate artifact SHA."
pushd ./web/output > /dev/null
declare checksum=$(find . -type f -exec md5sum {} + | awk '{print $1}' | \
sort | md5sum | awk '{print $1}')
popd > /dev/null
declare -r output_dir="deploy/${checksum}"
# Prep upload artifact.
echo "> Prep upload of artifact ${checksum}."
pushd ./web
mkdir -p $output_dir
cp *css $output_dir
cp *html $output_dir
cp output/* $output_dir
# Modify URLs in index.html
echo "> Modify asset paths, etc. in index.html."
pushd $output_dir
# /assets/output/main.built.js -> /assets/main.built.js
sed -i "s/assets\/output/assets/g" index.html
# /assets/main.built.js -> //cdn.trumped.com/assets/${checksum}/main.built.js
sed -i "s/\/assets/\/\/cdn.trumped.com\/assets\/${checksum}/g" index.html
# Variable substitutions
sed -i "s/asset_content_hash/${checksum}/g" index.html
sed -i "s/DEVELOPMENT/production/g" index.html
sed -i "s/API_HOST/http:\/\/api.trumped.com/g" index.html
sed -i "s/CDN_HOST/\/\/cdn.trumped.com/g" index.html
popd
aws s3 cp $output_dir s3://cdn.trumped.com/assets/${checksum} --recursive
aws s3 cp $output_dir/index.html s3://trumped.com/index.html
#aws s3 cp $output_dir/index.html s3://junglehorse.com/index.html
popd
}
# Upload files to S3 that infrequently change.
# Don't run this often as it adds to PutObject costs!
upload_stable_assets() {
echo '> Upload stable assets (images, robots.txt, etc.)'
pushd web
# S3 Bucket
# trumped.com website
aws s3 cp error.html s3://trumped.com/
aws s3 cp favicon.ico s3://trumped.com/favicon.ico
aws s3 cp robots.txt s3://trumped.com/
# S3 Bucket
# junglehorse.com website
#aws s3 cp error.html s3://junglehorse.com/
#aws s3 cp favicon.ico s3://junglehorse.com/favicon.ico
#aws s3 cp robots.txt s3://junglehorse.com/
# Cloudfront bucket.
# Keep error.html in both buckets.
aws s3 cp error.html s3://cdn.trumped.com/
aws s3 cp images/ s3://cdn.trumped.com/images/ --recursive
popd
}
upload_assets
upload_stable_assets
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
# Print a histogram of top monophones, diphones, triphones, and n-phones.
from os import path
import os
import re
DICTIONARY_FILE = '/home/bt/dev/trumpet/dictionary/cmudict-0.7b'
def phoneme_dictionary():
"""
Read the CMU Word-Phoneme dictionary and extract the phoneme map.
Returns a map of {word => monophone list}
"""
lines = []
with open(DICTIONARY_FILE) as f:
lines = f.readlines()
lines = [l for l in lines if re.match("^\w+", l)]
dictionary = {}
for l in lines:
word, *word_phones = l.split()
dictionary[word] = word_phones
return dictionary
def get_n_phones(phoneme_list, n):
"""
Return a list of n-phones within a phoneme list.
Returns a list of lists of strings or an empty list.
"""
if n < 1 or n > len(phoneme_list):
return []
elif n == len(phoneme_list):
return [phoneme_list[:]]
else:
span = len(phoneme_list) - n + 1
n_phones = []
for i in range(span):
n_phone = [phoneme_list[j] for j in range(i, i+n)]
n_phones.append(n_phone)
return n_phones
def print_histogram(n, print_length = 20):
"""
Build and print the histogram for the given size
"""
dictionary = phoneme_dictionary()
n_phone_map = {}
for word, phoneme in dictionary.items():
n_phones = get_n_phones(phoneme, n)
for n_phone in n_phones:
key = str(n_phone)
if key not in n_phone_map:
n_phone_map[key] = []
n_phone_map[key].append(word)
ordered = sorted(n_phone_map.items(), key=lambda x: len(x[1]), reverse=True)
title = "MOST FREQUENT {}-PHONES:".format(n)
print("\n" + title)
print("="*len(title) + "\n")
for it in ordered[0:print_length]:
line = "{}, {} - eg.{}".format(it[0], len(it[1]), it[1][0])
print(line)
print_histogram(2)
print_histogram(3)
print_histogram(4)
# Beyond 4-phones, it becomes computationally expensive
# if we try to guarantee best fit (exponential-time).
#print_histogram(5)
#print_histogram(6)
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
# Print a histogram of the longest polyphones in the database.
# Important for determining whether or not I want to do exponential
# algorithms.
from os import path
import os
import re
DICTIONARY_FILE = '/home/bt/dev/trumpet/dictionary/cmudict-0.7b'
def phoneme_dictionary():
"""
Read the CMU Word-Phoneme dictionary and extract the phoneme map.
Returns a map of {word => monophone list}
"""
lines = []
with open(DICTIONARY_FILE) as f:
lines = f.readlines()
lines = [l for l in lines if re.match("^\w+", l)]
dictionary = {}
for l in lines:
word, *word_phones = l.split()
dictionary[word] = word_phones
return dictionary
def print_length_histogram():
"""
Build and print the histogram of phoneme lengths
"""
dictionary = phoneme_dictionary()
histogram = {}
for word, phoneme in dictionary.items():
length = len(phoneme)
if length not in histogram:
histogram[length] = 0
histogram[length] += 1
ordered = sorted(histogram.items(), key=lambda x: x, reverse=True)
for size, count in ordered:
line = "{}: {}".format(size, count)
print(line)
print_length_histogram()
+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env python2
# Show how many phonemes remain
from os import path
import os
import re
DICTIONARY_FILE = '/home/bt/dev/trumpet/dictionary/cmudict-0.7b'
PHONE_DIR = '/home/bt/Dropbox/jungle.horse/sounds/trump/1-phones'
def current_phonemes():
"""
Gets the current set of phoneme wav files.
Returns a frozenset of strings.
"""
phonemes = []
for p in os.listdir(PHONE_DIR):
if not p.endswith('.wav'):
continue
phoneme = p.replace('.wav', '')
phonemes.append(phoneme)
return frozenset(phonemes)
def all_phonemes():
"""
Read the CMU Word-Phoneme dictionary and extract the phoneme set.
Returns a frozenset of strings.
"""
lines = []
with open(DICTIONARY_FILE) as f:
lines = f.readlines()
lines = [l for l in lines if re.match("^\w+", l)]
phonemes = set()
for l in lines:
word_phones = l.split()[1:]
for phoneme in word_phones:
phonemes.add(phoneme)
return frozenset(phonemes)
current = current_phonemes()
phonemes = all_phonemes()
print "\nAll Phonemes: {}\n".format(len(phonemes))
li = list(phonemes)
li.sort()
print li
print "\nCurrent Phonemes: {}\n".format(len(current))
li = list(current)
li.sort()
print li
missing = phonemes - current
print "\nMissing Phonemes: {}\n".format(len(missing))
li = list(missing)
li.sort()
print li
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
from urllib.parse import quote
import requests
class LoadtestClient:
def __init__(self, hostname, port = None):
if port:
host = "{}:{}".format(hostname, port)
else:
host = hostname
self.base_url = "http://{}/speak".format(host)
def make_url(self, sentence):
sentence = quote(sentence)
return "{}?v=trump&vol=3&s={}".format(self.base_url, sentence)
def request(self, sentence):
url = self.make_url(sentence)
print(url)
r = requests.get(url)
print(r.status_code)
def main():
client = LoadtestClient('localhost', 9000)
while True:
# TODO: Why is "HTTP" broken?
sentence = "Recreational use of other libraries may result in dangerous side-effects, including: security vulnerabilities, verbose code, reinventing the wheel, constantly reading documentation, depression, headaches, or even death."
client.request(sentence)
if __name__ == '__main__':
main()
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env python3
# Copyright 2016 Brandon Thomas <bt@brand.io>
# Increase the volume of wav files.
import numpy
import struct
import sys
import wave
SHRT_MIN = -32768
SHRT_MAX = 32767
# TODO: investigate `PyAudio`.
# numpy.fromstring(frames, numpy.int16) #/ 5 * 100
# new_frames = struct.pack('h'*len(frames), *frames)
def read_wav(filename):
"""
Read a wav file, return a tuple of (wav_params, frames_bytestring).
"""
params = None
frames = None
with wave.open(filename, 'rb') as w:
params = w.getparams()
numframes = params[3]
frames_bytestring = w.readframes(numframes)
return (params, frames_bytestring)
def write_wav(output_filename, wav_params, frames_bytestring):
"""Write a wav file."""
with wave.open(output_filename, 'wb') as w:
w.setparams(wav_params)
w.writeframes(frames_bytestring)
def generate_noise():
pass
def get_struct_format(wav_params, num_samples):
sample_width = wav_params[1]
if sample_width not in [1, 2]:
raise ValueError("Only supports 8 and 16 bit audio.")
if sample_width == 1:
# read unsigned chars
return '{}B'.format(num_samples)
else:
# read signed 2 byte shorts
return '{}h'.format(num_samples)
def get_pcm_channels(wav_params, frames):
"""
Decompose a wave bytestring (with its associated metadata) into its
constituent PCM data streams.
Returns a list of integer data streams (a stream per channel) as a
list of list of ints.
From http://stackoverflow.com/a/2227174
"""
sample_width = wav_params[1]
if sample_width not in [1, 2]:
raise ValueError("Only supports 8 and 16 bit audio.")
num_channels = wav_params[0]
num_frames = wav_params[3]
total_samples = num_frames * num_channels
fmt = get_struct_format(wav_params, total_samples)
integer_data = struct.unpack(fmt, frames)
channels = [[] for i in range(num_channels)]
for i, value in enumerate(integer_data):
bucket = i % num_channels
channels[bucket].append(value)
return channels
def channels_to_bytestring(wav_params, pcm_data):
sample_width = wav_params[1]
num_channels = wav_params[0]
num_frames = wav_params[3]
assert(num_channels == len(pcm_data))
assert(num_frames == len(pcm_data[0]))
integer_data = []
for j in range(num_frames):
for i in range(num_channels):
integer_data.append(pcm_data[i][j])
print("Length of pcm_data: {}".format(len(pcm_data)))
print("Length of a pcm channel: {}".format(len(pcm_data[0])))
print("Length of packed integer data: {}".format(len(integer_data)))
total_samples = num_frames * num_channels
fmt = get_struct_format(wav_params, total_samples)
return struct.pack(fmt, *integer_data)
def increase_volume(pcm_data, multiplier=2):
"""
Increase the volume of the PCM data.
"""
num_channels = len(pcm_data)
num_frames = len(pcm_data[0])
for j in range(num_frames):
for i in range(num_channels):
x = pcm_data[i][j]
# Change volume by changing the signal amplitude!
x *= multiplier
# Don't overflow.
pcm_data[i][j] = saturate_signed_short(x)
def saturate_signed_short(integer):
"""Don't overflow the signed short width."""
if integer > SHRT_MAX:
return SHRT_MAX
elif integer < SHRT_MIN:
return SHRT_MIN
return int(integer)
def main():
if len(sys.argv) < 3:
print("Call as ${} INPUT_FILE OUTPUT_FILE".format(sys.argv[0]))
infile = sys.argv[1]
outfile = sys.argv[2]
multiplier = 2
if len(sys.argv) > 3:
multiplier = int(sys.argv[3])
params, frames = read_wav(infile)
print(params)
channels = get_pcm_channels(params, frames)
print('Volume multiplier: {}'.format(multiplier))
increase_volume(channels, multiplier)
bytestring = channels_to_bytestring(params, channels)
print("Length of original bytestring: {}".format(len(frames)))
print("Length of new bytestring: {}".format(len(bytestring)))
write_wav(outfile, params, bytestring)
if __name__ == '__main__':
main()
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
#
# Includes code from:
# http://www.frank-zalkow.de/en/code-snippets/
# create-audio-spectrograms-with-python.html
import sys
import numpy as np
import matplotlib.pyplot as plt
import scipy.io.wavfile as wav
from numpy.lib import stride_tricks
""" short time fourier transform of audio signal """
def stft(sig, frameSize, overlapFac=0.5, window=np.hanning):
win = window(frameSize)
hopSize = int(frameSize - np.floor(overlapFac * frameSize))
# zeros at beginning (thus center of 1st window should be for sample nr. 0)
samples = np.append(np.zeros(np.floor(frameSize/2.0)), sig)
# cols for windowing
cols = np.ceil( (len(samples) - frameSize) / float(hopSize)) + 1
# zeros at end (thus samples can be fully covered by frames)
samples = np.append(samples, np.zeros(frameSize))
frames = stride_tricks.as_strided(samples,
shape=(cols, frameSize),
strides=(samples.strides[0]*hopSize, samples.strides[0])).copy()
frames *= win
return np.fft.rfft(frames)
""" scale frequency axis logarithmically """
def logscale_spec(spec, sr=44100, factor=20.):
timebins, freqbins = np.shape(spec)
scale = np.linspace(0, 1, freqbins) ** factor
scale *= (freqbins-1)/max(scale)
scale = np.unique(np.round(scale))
# create spectrogram with new freq bins
newspec = np.complex128(np.zeros([timebins, len(scale)]))
for i in range(0, len(scale)):
if i == len(scale)-1:
newspec[:,i] = np.sum(spec[:,scale[i]:], axis=1)
else:
newspec[:,i] = np.sum(spec[:,scale[i]:scale[i+1]], axis=1)
# list center freq of bins
allfreqs = np.abs(np.fft.fftfreq(freqbins*2, 1./sr)[:freqbins+1])
freqs = []
for i in range(0, len(scale)):
if i == len(scale)-1:
freqs += [np.mean(allfreqs[scale[i]:])]
else:
freqs += [np.mean(allfreqs[scale[i]:scale[i+1]])]
return newspec, freqs
""" plot spectrogram"""
def plotstft(audiopath, binsize=2**10, plotpath=None, colormap="jet"):
samplerate, samples = wav.read(audiopath)
s = stft(samples, binsize)
sshow, freq = logscale_spec(s, factor=1.0, sr=samplerate)
ims = 20.*np.log10(np.abs(sshow)/10e-6) # amplitude to decibel
timebins, freqbins = np.shape(ims)
plt.figure(figsize=(15, 7.5))
plt.imshow(np.transpose(ims), origin="lower", aspect="auto",
cmap=colormap, interpolation="none")
plt.colorbar()
plt.xlabel("time (s)")
plt.ylabel("frequency (hz)")
plt.xlim([0, timebins-1])
plt.ylim([0, freqbins])
xlocs = np.float32(np.linspace(0, timebins-1, 5))
plt.xticks(xlocs, ["%.02f" % l for l in (
(xlocs*len(samples)/timebins)+(0.5*binsize))/samplerate])
ylocs = np.int16(np.round(np.linspace(0, freqbins-1, 10)))
plt.yticks(ylocs, ["%.02f" % freq[i] for i in ylocs])
if plotpath:
plt.savefig(plotpath, bbox_inches="tight")
else:
plt.show()
plt.clf()
def main():
if len(sys.argv) < 2:
print("Call as ${} INPUT_FILE".format(sys.argv[0]))
infile = sys.argv[1]
plotstft(infile)
if __name__ == '__main__':
main()
View File
Whitespace-only changes.
+162
View File
@@ -0,0 +1,162 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
use resolve::hostname;
use std::convert::From;
use std::fs::File;
use std::io::Error;
use std::io::Read;
use std::io;
use toml;
#[derive(Clone, Debug, RustcEncodable, RustcDecodable)]
pub struct Config {
/// Whether or not to send/respect client caching headers.
pub use_caching_headers: Option<bool>,
/// Where sound files are located.
pub sound_path: Option<String>,
pub html_path: Option<String>,
/// Where various arpabet files are located.
pub phoneme_dictionary_file: Option<String>,
pub arpabet_override_dictionary_file: Option<String>,
pub extra_dictionary_file: Option<String>,
pub square_dictionary_file: Option<String>,
/// Where the abbreviation file is found.
pub abbreviation_file: Option<String>,
/// Input control. Respond with 400 when over limit.
pub max_num_characters: Option<usize>,
pub max_num_words: Option<usize>,
/// What events to log.
pub log_handler: Option<bool>,
pub log_parsing: Option<bool>,
pub log_raw_sentence: Option<bool>,
pub log_syllable_decomposition: Option<bool>,
pub log_synthesis: Option<bool>,
pub log_timing: Option<bool>,
}
#[derive(Debug)]
pub enum ConfigError {
/// Wraps an IO error.
IoError { cause: io::Error },
/// TOML parsing error.
TomlError,
/// Configs absent.
TomlMissing,
}
impl Config {
/// Static CTOR.
/// Read a [default] set of configs and merge with a ["$hostname"] set.
pub fn read(filename: &str) -> Result<Config, ConfigError> {
let contents = read_file(filename)?;
let table = match toml::Parser::new(&contents).parse() {
None => { return Err(ConfigError::TomlError); },
Some(table) => table,
};
let system_hostname = hostname::get_hostname().ok();
let maybe_host = if system_hostname.is_some() {
table.get(&system_hostname.unwrap())
.and_then(|t| toml::decode::<Config>(t.clone()))
} else {
None
};
let maybe_default = table.get("default")
.and_then(|t| toml::decode::<Config>(t.clone()));
if maybe_default.is_none() {
if maybe_host.is_none() {
return Err(ConfigError::TomlMissing);
} else {
return Ok(maybe_host.unwrap());
}
}
let default = maybe_default.unwrap();
if maybe_host.is_some() {
let host = maybe_host.unwrap();
Ok(host.merge(default))
} else {
Ok(default)
}
}
/// Merge another config object, keeping current values where they
/// exist, and overriding Optional values where they do not.
pub fn merge(&self, other: Config) -> Config {
// TODO: Simplify using generics. Maybe a trait, such as "x.as_default_or(y)"
Config {
use_caching_headers: self.use_caching_headers
.clone()
.or(other.use_caching_headers.clone()),
sound_path: self.sound_path
.clone()
.or(other.sound_path.clone()),
html_path: self.html_path
.clone()
.or(other.html_path.clone()),
phoneme_dictionary_file: self.phoneme_dictionary_file
.clone()
.or(other.phoneme_dictionary_file.clone()),
arpabet_override_dictionary_file: self.arpabet_override_dictionary_file
.clone()
.or(other.arpabet_override_dictionary_file.clone()),
extra_dictionary_file: self.extra_dictionary_file
.clone()
.or(other.extra_dictionary_file.clone()),
square_dictionary_file: self.square_dictionary_file
.clone()
.or(other.square_dictionary_file.clone()),
abbreviation_file: self.abbreviation_file
.clone()
.or(other.abbreviation_file.clone()),
max_num_characters: self.max_num_characters
.clone()
.or(other.max_num_characters.clone()),
max_num_words: self.max_num_words
.clone()
.or(other.max_num_words.clone()),
log_handler: self.log_handler
.clone()
.or(other.log_handler.clone()),
log_parsing: self.log_parsing
.clone()
.or(other.log_parsing.clone()),
log_raw_sentence: self.log_raw_sentence
.clone()
.or(other.log_raw_sentence.clone()),
log_syllable_decomposition: self.log_syllable_decomposition
.clone()
.or(other.log_syllable_decomposition.clone()),
log_synthesis: self.log_synthesis
.clone()
.or(other.log_synthesis.clone()),
log_timing: self.log_timing
.clone()
.or(other.log_timing.clone()),
}
}
}
impl From<io::Error> for ConfigError {
fn from(error: io::Error) -> ConfigError {
ConfigError::IoError { cause: error }
}
}
fn read_file(filename: &str) -> Result<String, Error> {
let mut file = File::open(filename)?;
let mut buf = String::new();
file.read_to_string(&mut buf)?;
Ok(buf)
}
+54
View File
@@ -0,0 +1,54 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
use hound::Error as HoundError;
use std::convert::From;
use std::error::Error;
use std::fmt;
use std::io;
/** The common error type used throughout the synthesizer. */
#[derive(Debug)]
pub enum SynthError {
/// Bad user input.
BadInput { description: &'static str },
/// The file was empty or did not have parsable contents.
EmptyFile,
/// Could not obtain the lock.
LockError,
/// Wraps an IoError.
IoError { cause: io::Error },
/// Wraps a Hound audio library error.
AudioError { cause: HoundError },
}
impl fmt::Display for SynthError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "Synthesis Error")
}
}
impl Error for SynthError {
fn description(&self) -> &str {
"Synthesis Error" // TODO
}
/*fn cause(&self) -> Option<&Error> {
Some(&self)
}*/
}
impl From<io::Error> for SynthError {
fn from(error: io::Error) -> SynthError {
SynthError::IoError { cause: error }
}
}
impl From<HoundError> for SynthError {
fn from(error: HoundError) -> SynthError {
SynthError::AudioError { cause: error }
}
}
+428
View File
@@ -0,0 +1,428 @@
// Copyright (c) 2015 Brandon Thomas <bt@brand.io>
use config::Config;
use crypto::digest::Digest;
use crypto::sha1::Sha1;
use iron::Handler;
use iron::IronError;
use iron::Plugin;
use iron::headers::ETag;
use iron::headers::EntityTag;
use iron::headers::IfNoneMatch;
use iron::mime::Mime;
use iron::prelude::IronResult;
use iron::prelude::Request;
use iron::prelude::Response;
use iron::status::BadRequest;
use iron::status;
use lang::parser::Parser;
use speaker::Speaker;
use std::collections::HashMap;
use std::i16;
use std::str::FromStr;
use std::sync::Arc;
use std::sync::RwLock;
use super::error_filter::build_error;
use synthesis::error::SynthesisError;
use synthesis::synthesizer::SynthesisParams;
use synthesis::synthesizer::Synthesizer;
use time::PreciseTime;
use urlencoded::UrlEncodedQuery;
const SENTENCE_PARAM : &'static str = "s";
const SPEAKER_PARAM : &'static str = "v";
const SPEED_PARAM : &'static str = "spd";
const USE_MONOPHONES_PARAM: &'static str = "um";
const USE_SYLLABLES_PARAM: &'static str = "us";
const USE_N_PHONES_PARAM: &'static str = "un";
const USE_WORDS_PARAM: &'static str = "uw";
const USE_ENDS_PARAM: &'static str = "ue";
const VOLUME_PARAM : &'static str = "vol";
const PADDING_BETWEEN_PHONES_PARAM: &'static str = "pbp";
const POLYPHONE_PADDING_START_PARAM : &'static str = "pps";
const POLYPHONE_PADDING_END_PARAM : &'static str = "ppe";
const WORD_PADDING_START_PARAM : &'static str = "wps";
const WORD_PADDING_END_PARAM : &'static str = "wpe";
type QueryParams = HashMap<String, Vec<String>>;
/// Represents a request to this endpoint.
#[derive(Clone, Debug)]
struct SpeakRequest {
/** The sentence to be spoken. */
pub sentence: String,
/** The voice to use to render the audio. */
pub speaker: Speaker,
/** An optional volume multiplier. */
pub volume: Option<f32>,
/** An optional speed multiplier. */
pub speed: Option<f32>,
/** Whether to use words. */
pub use_words: bool,
/** Whether to use n-phones. */
pub use_n_phones: bool,
/** Whether to use bare monophones. */
pub use_bare_monophones: bool,
/** Whether to use syllable boundaries. */
pub use_syllables: bool,
/** Whether to use "ends": start, end, etc. */
pub use_ends: bool,
/** Padding between monophones or n-phone clusters. */
pub padding_between_phones: Option<u16>,
/** Padding before a polyphone. */
pub polyphone_padding_start: Option<u16>,
/** Padding after a polyphone. */
pub polyphone_padding_end: Option<u16>,
/// Silent padding before a word (not one constructed from phones).
pub word_padding_start: Option<u16>,
/// Silent padding after a word (not one constructed from phones).
pub word_padding_end: Option<u16>,
}
enum SpeakerRequestError {
//SentenceMissing,
SentenceInvalid,
//SpeakerMissing,
SpeakerInvalid,
//VolumeInvalid,
}
/// Synthesizes audio from input.
pub struct AudioSynthHandler {
/// Sentence parser.
parser: Parser,
/// The TTS synthesizer.
synthesizer: Arc<RwLock<Synthesizer>>,
/// Server configs.
config: Config,
}
impl SpeakRequest {
/// Parse a speak request from a raw HTTP request.
pub fn parse(http_request: &mut Request)
-> Result<SpeakRequest, SpeakerRequestError> {
let params = match http_request.get_ref::<UrlEncodedQuery>() {
Ok(multimap) => multimap,
Err(_) => {
return Err(SpeakerRequestError::SentenceInvalid);
},
};
let sen = match get_str(params, SENTENCE_PARAM) {
Some(s) => s.to_string(),
None => {
return Err(SpeakerRequestError::SentenceInvalid);
},
};
let spk = match get_str(params, SPEAKER_PARAM) {
Some(s) => Speaker::new(s.to_string()),
None => {
return Err(SpeakerRequestError::SpeakerInvalid);
},
};
let volume = get_f32(params, VOLUME_PARAM);
let speed = get_f32(params, SPEED_PARAM).and_then(|f| {
// Don't waste CPU calculating speed if it isn't supplied.
let diff = f - 1.0;
if diff > -0.005 && diff < 0.005 {
None
} else {
Some(f)
}
});
let use_monophones = get_bool(params, USE_MONOPHONES_PARAM).unwrap_or(true);
let use_n_phones = get_bool(params, USE_N_PHONES_PARAM).unwrap_or(true);
let use_syllables = get_bool(params, USE_SYLLABLES_PARAM).unwrap_or(true); // new hotness
let use_words = get_bool(params, USE_WORDS_PARAM).unwrap_or(true);
let use_ends = get_bool(params, USE_ENDS_PARAM).unwrap_or(true);
let pbp = get_u16(params, PADDING_BETWEEN_PHONES_PARAM).or(None);
let pps = get_u16(params, POLYPHONE_PADDING_START_PARAM).or(Some(600));
let ppe = get_u16(params, POLYPHONE_PADDING_END_PARAM).or(Some(600));
let wps = get_u16(params, WORD_PADDING_START_PARAM).or(Some(600));
let wpe = get_u16(params, WORD_PADDING_END_PARAM).or(Some(600));
Ok(SpeakRequest {
sentence: sen,
speaker: spk,
volume: volume,
speed: speed,
use_bare_monophones: use_monophones,
use_n_phones: use_n_phones,
use_syllables: use_syllables,
use_words: use_words,
use_ends: use_ends,
padding_between_phones: pbp,
polyphone_padding_start: pps,
polyphone_padding_end: ppe,
word_padding_start: wps,
word_padding_end: wpe,
})
}
/// Generate a SHA checksum of the request.
/// This can be used as a caching header for the client.
fn sha_digest(&self) -> String {
let mut hasher = Sha1::new();
hasher.input_str(&self.speaker.to_string());
hasher.input_str(&self.sentence);
if self.volume.is_some() {
let vol = self.volume.unwrap();
// This isn't perfect hashing for floats, but is mostly
// what I want.
let hashed = (vol * 1000.0) as i16;
let hi = (hashed >> 8 & 0xff) as u8;
let lo = (hashed & 0xff) as u8;
hasher.input(&[hi, lo]);
} else {
hasher.input(&[0u8]);
}
let mut use_byte = 0u8;
if self.use_bare_monophones { use_byte |= 1 << 1; }
if self.use_n_phones { use_byte |= 1 << 2; }
if self.use_syllables { use_byte |= 1 << 3; }
if self.use_words { use_byte |= 1 << 4; }
if self.use_ends { use_byte |= 1 << 5; }
hasher.input(&[use_byte]);
hasher.result_str()
}
}
impl Handler for AudioSynthHandler {
/// Process request.
fn handle(&self, req: &mut Request) -> IronResult<Response> {
let sentence_error = build_error(BadRequest,
&format!("Missing `{}` parameter.", SENTENCE_PARAM));
let request = match SpeakRequest::parse(req) {
Err(_) => { return sentence_error },
Ok(s) => s,
};
info!(target: "handler", "Speak Request: {:?}", request);
info!(target: "raw_sentence", "{:?}", request.sentence);
// Get the request ETag. TODO: Cleanup
let request_hash = {
match req.headers.get::<IfNoneMatch>() {
None => { "".to_string() },
Some(etag) => { etag.to_string() }
}
};
// FIXME: Varies with spaces, formatting, etc.
let hash = request.sha_digest();
let entity_tag = EntityTag::new(true, hash.to_owned());
info!(target: "handler", "Request Header Caching Sha: {}", hash);
if self.config.use_caching_headers.unwrap_or(true) {
// Don't generate file if caching header is matched.
if request_hash == entity_tag.to_string() {
info!(target: "handler",
"Caching headers match; responding with NotModified.");
return Ok(Response::with(status::NotModified));
}
}
let max_characters = self.config.max_num_characters.unwrap_or(5_000);
if request.sentence.len() > max_characters {
return build_error(BadRequest, "Too many characters in request.");
}
// GENERATE AUDIO
let start = PreciseTime::now();
let synth = self.synthesizer.read().map_err(|_| SynthesisError::LockError)?;
let params = SynthesisParams {
use_words: request.use_words,
use_bare_monophones: request.use_bare_monophones,
use_n_phones: request.use_n_phones,
use_syllables: request.use_syllables,
use_ends: request.use_ends,
volume: request.volume,
speed: request.speed,
padding_between_phones: request.padding_between_phones,
polyphone_padding_start: request.polyphone_padding_start,
polyphone_padding_end: request.polyphone_padding_end,
word_padding_start: request.word_padding_start,
word_padding_end: request.word_padding_end,
};
let tokens = self.parser.parse(&request.sentence);
if tokens.len() > self.config.max_num_words.unwrap_or(1_000) {
return build_error(BadRequest, "Too many words in request.");
}
let result = synth.generate(tokens, &request.speaker, params)?;
info!(target: "timing",
"Total parsing and synthesis took: {}", start.to(PreciseTime::now()));
let mime_type = "audio/wav".parse::<Mime>().unwrap();
let mut response = Response::with((mime_type, status::Ok, result));
response.headers.set(ETag(entity_tag));
Ok(response)
}
}
impl AudioSynthHandler {
pub fn new(parser: Parser,
synthesizer: Arc<RwLock<Synthesizer>>,
config: Config) -> AudioSynthHandler {
AudioSynthHandler {
parser: parser,
synthesizer: synthesizer,
config: config,
}
}
}
fn get_str<'a>(params: &'a QueryParams, param_name: &str)
-> Option<&'a String> {
params.get(param_name)
.and_then(|v| v.get(0))
}
fn get_bool(params: &QueryParams, param_name: &str) -> Option<bool> {
get_str(params, param_name)
.map(|s| s.trim())
.map(|s| FromStr::from_str(s))
.and_then(|res| res.ok())
}
fn get_u16(params: &QueryParams, param_name: &str) -> Option<u16> {
get_str(params, param_name)
.map(|s| s.trim())
.map(|s| s.parse::<u16>())
.and_then(|res| res.ok())
.and_then(|d| if d == 0 { None } else { Some(d) } )
}
fn get_f32(params: &QueryParams, param_name: &str) -> Option<f32> {
get_str(params, param_name)
.map(|s| s.trim())
.map(|s| s.parse::<f32>())
.and_then(|res| res.ok())
}
impl From<SynthesisError> for IronError {
fn from(error: SynthesisError) -> IronError {
let mime = "text/plain".parse::<Mime>().unwrap();
let response = (mime, status::InternalServerError, "Service Error");
IronError::new(error, response)
}
}
#[cfg(test)]
mod tests {
use speaker::Speaker;
use super::SpeakRequest;
#[test]
fn test_request_hash() {
let request_1 = SpeakRequest {
sentence: "same sentence".to_string(),
speaker: Speaker::new("speaker".to_string()),
volume: Some(2.0),
speed: Some(1.5),
use_bare_monophones: true,
use_n_phones: false,
use_syllables: true,
use_words: false,
use_ends: false,
padding_between_phones: Some(300),
polyphone_padding_start: None,
polyphone_padding_end: None,
word_padding_start: None,
word_padding_end: None,
};
let request_2 = SpeakRequest {
sentence: "same sentence".to_string(),
speaker: Speaker::new("speaker".to_string()),
volume: Some(2.0),
speed: Some(1.5),
use_bare_monophones: true,
use_n_phones: false,
use_syllables: true,
use_words: false,
use_ends: false,
padding_between_phones: Some(300),
polyphone_padding_start: None,
polyphone_padding_end: None,
word_padding_start: None,
word_padding_end: None,
};
assert!(request_1.sha_digest() == request_2.sha_digest());
let request_2 = SpeakRequest {
sentence: "same sentence".to_string(),
speaker: Speaker::new("speaker".to_string()),
volume: None,
speed: Some(1.5),
use_bare_monophones: false,
use_n_phones: true,
use_syllables: false,
use_words: true,
use_ends: false,
padding_between_phones: Some(300),
polyphone_padding_start: None,
polyphone_padding_end: None,
word_padding_start: None,
word_padding_end: Some(400),
};
assert!(request_1.sha_digest() != request_2.sha_digest());
let request_2 = SpeakRequest {
sentence: "different sentence".to_string(),
speaker: Speaker::new("speaker".to_string()),
volume: Some(2.0),
speed: Some(1.5),
use_bare_monophones: true,
use_n_phones: false,
use_syllables: true,
use_words: false,
use_ends: false,
padding_between_phones: Some(300),
polyphone_padding_start: None,
polyphone_padding_end: None,
word_padding_start: None,
word_padding_end: None,
};
assert!(request_1.sha_digest() != request_2.sha_digest());
}
}
+46
View File
@@ -0,0 +1,46 @@
// Copyright (c) 2015 Brandon Thomas <bt@brand.io>
use iron::AfterMiddleware;
use iron::prelude::*;
use iron::status::Status;
use std::error::Error;
use std::fmt::{self, Debug};
/// Error-handling filter.
pub struct ErrorFilter;
/// An error type compatible with IronError.
#[derive(Debug)]
pub struct StringError(pub String);
pub fn build_error(status: Status, message: &str) -> IronResult<Response> {
Err(IronError::new(StringError(message.to_string()), status))
}
impl AfterMiddleware for ErrorFilter {
fn catch(&self, _: &mut Request, err: IronError) -> IronResult<Response> {
error!(target: "handler", "Error has occurred! Error: {}", err.error);
// TODO: Return JSON if request was JSON, otherwise HTML.
match err.response.status {
Some(status) => {
Ok(Response::with((status, err.error.description())))
},
_ => Err(err)
}
}
}
impl fmt::Display for StringError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
Debug::fmt(self, f)
}
}
impl Error for StringError {
fn description(&self) -> &str {
&*self.0
}
}
+120
View File
@@ -0,0 +1,120 @@
// Copyright (c) 2015 Brandon Thomas <bt@brand.io>
use iron::Handler;
use iron::mime::Mime;
use iron::prelude::*;
use iron::status;
use router::Router;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};
// TODO: Make this secure! Mimetypes, limit to single directory root,
// sanitize input.
/// FileServerHandler is used to serve files. It is *NOT* secure.
pub struct FileServerHandler {
/// Root of where files can be served from.
/// A PathBuf since `Path` can only be created as a borrow.
file_root: PathBuf,
/// Default file to route to if none is found.
index_file: String,
}
impl Handler for FileServerHandler {
/// Handles routes of the form `/f/:fileame`
fn handle(&self, req: &mut Request) -> IronResult<Response> {
info!(target: "handler", "Request: {}", &req.url);
let filename = self.get_filename(&req);
let mime = match &filename {
s if s.ends_with(".css") => "text/css",
s if s.ends_with(".html") => "text/html",
s if s.ends_with(".js") => "application/javascript",
s if s.ends_with(".txt") => "text/plain",
_ => "text/plain",
};
match self.open_file(&filename) {
None => {
Ok(Response::with((status::NotFound, "File not found.")))
},
Some(contents) => {
let content_type = mime.parse::<Mime>().unwrap();
Ok(Response::with((content_type, status::Ok, contents)))
},
}
}
}
impl FileServerHandler {
pub fn new(file_root: &str, index_file: &str) -> FileServerHandler {
FileServerHandler {
file_root: Path::new(file_root).to_path_buf(),
index_file: index_file.to_string(),
}
}
// FIXME: This is dangerous and horrible code
/// Get the filename requested.
fn get_filename(&self, request: &Request) -> String {
if request.url.path().len() == 1
&& request.url.path()[0] == "test".to_string() {
// Route `/test`
"test.html".to_string()
} else if request.url.path().len() > 2
&& request.url.path()[1] == "output".to_string() {
// Route `/assets/*/*`
let mut path = String::new();
for i in 1..request.url.path().len() {
let part = &request.url.path()[i];
if i == 1 {
path = path + &part;
} else {
path = path + "/" + &part;
}
}
path
} else {
// Other routes or default
request.extensions.get::<Router>().unwrap()
.find("filename")
.unwrap_or(&self.index_file)
.to_string()
}
}
// FIXME: This is not secure.
fn open_file(&self, request_filename: &str) -> Option<String> {
let path = Path::new(request_filename);
// TODO: Dangerous to omit this
/*let filename = match path.file_name() {
None => { return None; },
Some(f) => { f },
};*/
let mut full_filename = self.file_root.clone();
// TODO: `push` security; abspath might replace
full_filename.push(path);
// TODO: whitelist filetypes.
// TODO: only open non-executable files.
debug!(target: "handler", "Opening file `{}`", full_filename.display());
let mut file = match File::open(full_filename) {
Err(_) => { return None },
Ok(f) => { f },
};
let mut contents = String::new();
match file.read_to_string(&mut contents) {
Err(_) => { None },
Ok(_) => { Some(contents) }
}
}
}
+73
View File
@@ -0,0 +1,73 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
use config::Config;
use iron::Handler;
use iron::error::HttpError;
use iron::error::IronError;
use iron::mime::Mime;
use iron::prelude::IronResult;
use iron::prelude::Request;
use iron::prelude::Response;
use iron::status::Status::InternalServerError;
use iron::status;
use std::fs::File;
use std::io::Error as IoError;
use std::io::Read;
use std::path::Path;
/// A handler that serves statically-defined routes to HTML files.
/// Not very flexible, but serves a purpose.
pub struct HtmlHandler {
config: Config,
}
lazy_static! {
static ref MIMETYPE: Mime = "text/html".parse::<Mime>().unwrap();
}
impl Handler for HtmlHandler {
fn handle(&self, req: &mut Request) -> IronResult<Response> {
let filename = match self.match_file(req) {
None => { return Ok(Response::with((status::NotFound, "File not found."))); },
Some(filename) => filename,
};
match self.read_file(filename) {
Err(err) => Err(IronError::new(HttpError::Io(err), InternalServerError)),
Ok(contents) => Ok(Response::with((MIMETYPE.clone(), status::Ok, contents))),
}
}
}
impl HtmlHandler {
pub fn new(config: Config) -> HtmlHandler {
HtmlHandler { config: config }
}
fn match_file(&self, request: &Request) -> Option<&'static str> {
let path = request.url.path().join("/");
match path.as_ref() {
"" => Some("index.html"),
"demo/jon" => Some("demo_jon.html"),
"demo/jt" => Some("demo_jt.html"),
"old" => Some("old.html"),
_ => None,
}
}
fn read_file(&self, filename: &str) -> Result<String, IoError> {
let mut path = self.config.html_path
.as_ref()
.and_then(|p| Some(Path::new(&p).to_path_buf()))
.unwrap_or_else(|| Path::new("./www").to_path_buf());
path.push(filename);
let mut file = File::open(&path)?;
let mut contents = String::new();
file.read_to_string(&mut contents)?;
Ok(contents)
}
}
+8
View File
@@ -0,0 +1,8 @@
// Copyright (c) 2015 Brandon Thomas <bt@brand.io>
pub mod audio_synth_handler;
pub mod error_filter;
pub mod file_server_handler;
pub mod html_handler;
pub mod vocab_list_handler;
+46
View File
@@ -0,0 +1,46 @@
// Copyright (c) 2015 Brandon Thomas <bt@brand.io>
use iron::Handler;
use iron::mime::Mime;
use iron::prelude::*;
use iron::status;
use old_dictionary::VocabularyLibrary;
use rustc_serialize::json;
use std::path::Path;
use std::path::PathBuf;
#[derive(RustcDecodable, RustcEncodable)]
struct WordsResponse {
pub words: Vec<String>
}
/// Returns a list of the available vocabulary.
pub struct VocabListHandler {
/// Root of where files can be served from.
/// A PathBuf since `Path` can only be created as a borrow.
directory: PathBuf,
}
impl Handler for VocabListHandler {
fn handle(&self, _req: &mut Request) -> IronResult<Response> {
// FIXME: This is inefficient to re-read on every request.
match VocabularyLibrary::read_from_directory(
self.directory.as_path()) {
Err(_) => {
Ok(Response::with((status::InternalServerError, "{\"error\": true}")))
},
Ok(library) => {
let response = json::encode(&library).unwrap();
let mime_type = "application/json".parse::<Mime>().unwrap();
Ok(Response::with((mime_type, status::Ok, response)))
}
}
}
}
impl VocabListHandler {
pub fn new(directory: &str) -> VocabListHandler {
VocabListHandler { directory: Path::new(directory).to_path_buf() }
}
}
+81
View File
@@ -0,0 +1,81 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
use error::SynthError;
use regex::Regex;
use std::collections::HashMap;
use std::fs::File;
use std::io::BufRead;
use std::io::BufReader;
pub struct AbbreviationsMap {
/// A map of abbreviation to the words the abbreviation stands for.
/// eg. 'omg' -> ["oh", "my", "god"]
dictionary: HashMap<String , Vec<String>>
}
impl AbbreviationsMap {
/// Primarily for testing.
pub fn empty() -> AbbreviationsMap {
AbbreviationsMap { dictionary: HashMap::new() }
}
/// Primarily for testing.
pub fn new(map: HashMap<String, Vec<String>>) -> AbbreviationsMap {
AbbreviationsMap { dictionary: map }
}
/// Read custom abbreviation dictionary.
/// Based on the CMU Arpabet dictionary format.
pub fn load_from_file(filename: &str) -> Result<AbbreviationsMap, SynthError> {
let f = File::open(filename)?;
let mut reader = BufReader::new(f);
let mut map = HashMap::new();
let mut buffer = String::new();
// Format resembles the following,
// "OMG oh my god"
let re = Regex::new(r"^([\w-'/]+)\s+(.*)\n$").unwrap();
while reader.read_line(&mut buffer)? > 0 {
match re.captures(&buffer) {
None => {},
Some(caps) => {
let abbr_match = caps.at(1);
let words_match = caps.at(2);
if abbr_match.is_some() && words_match.is_some() {
let abbr = abbr_match.unwrap().to_lowercase();
let split = words_match.unwrap().split(" ");
let v1 = split.collect::<Vec<&str>>();
let v2 = v1.iter()
.map(|s| s.to_lowercase().to_string())
.collect::<Vec<String>>();
map.insert(abbr, v2);
}
},
}
buffer.clear();
}
if map.is_empty() {
Err(SynthError::EmptyFile)
} else {
Ok(AbbreviationsMap { dictionary: map })
}
}
/// Whether the abbreviation exists.
pub fn is_abbreviation(&self, abbreviation: &str) -> bool {
self.dictionary.contains_key(abbreviation)
}
/// Get an abbreviation from the map.
pub fn get_words(&self, abbreviation: &str) -> Option<Vec<String>> {
self.dictionary.get(abbreviation).and_then(|p| {
Some(p.iter().map(|s| s.to_string()).collect::<Vec<String>>())
})
}
}
+372
View File
@@ -0,0 +1,372 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
use lang::numbers::number_to_words;
use lang::ordinals::ORDINAL_MAP;
/**
* Convert a date of the format N+/N+/NN+ or N+/NN+ to its word representation.
*/
pub fn date_to_words(date: &str) -> Option<Vec<String>> {
let split: Vec<&str> = date.split("/").collect();
// US date order
let mut month = None;
let mut day = None;
let mut year = None;
match split.len() {
// Mon/Day: 7/11
// Mon/Year: 10/2016
2 => {
month = Some(split[1]);
match split[1].len() {
2 => { day = Some(split[1]); },
4 => { year = Some(split[1]); },
_ => { return None; },
}
},
// Mon/Day/Year: 10/7/2016
// Mon/Day/Year: 10/7/16
3 => {
month = Some(split[0]);
day = Some(split[1]);
year = Some(split[2]);
},
_ => {},
}
let mut words = Vec::new();
if month.is_some() {
match month_to_words(month.unwrap()) {
None => { return None; },
Some(m) => {
words.push(m.to_string());
}
}
}
if day.is_some() {
match day_to_words(day.unwrap()) {
None => { return None; },
Some(mut d) => {
words.append(&mut d);
}
}
}
if year.is_some() {
match year_to_words(year.unwrap()) {
None => { return None; },
Some(mut y) => {
words.append(&mut y);
}
}
}
if words.len() == 0 {
None
} else {
Some(words)
}
}
/**
* Convert a month to string.
*/
pub fn month_to_words(month: &str) -> Option<&'static str> {
match month {
"1" | "01" => Some("january"),
"2" | "02" => Some("february"),
"3" | "03" => Some("march"),
"4" | "04" => Some("april"),
"5" | "05" => Some("may"),
"6" | "06" => Some("june"),
"7" | "07" => Some("july"),
"8" | "08" => Some("august"),
"9" | "09" => Some("september"),
"10" => Some("october"),
"11" => Some("november"),
"12" => Some("december"),
_ => None,
}
}
/**
* Convert a day to string.
*/
pub fn day_to_words(day: &str) -> Option<Vec<String>> {
let nonzero_day = day.trim_left_matches("0");
let num = match nonzero_day.parse::<i64>() {
Err(_) => { return None; },
Ok(num) => num,
};
if num < 1 || num > 31 {
return None;
}
let mut words = match number_to_words(num) {
None => { return None; },
Some(words) => { words },
};
match words.last_mut() {
None => { return None; },
Some(mut last) => {
// NB: Tell the compiler we want str.
match ORDINAL_MAP.get::<str>(&last) {
None => { return None; },
Some(ordinal) => { *last = ordinal.to_string(); },
}
},
}
Some(words)
}
/**
* Convert a year of the format YY or YYYY to word representation.
* This heuristically pins two digit years to the range [1970, 2069].
*/
pub fn year_to_words(year: &str) -> Option<Vec<String>> {
// FIXME: Efficiency and clarity.
let num = match year.parse::<i64>() {
Err(_) => { return None; },
Ok(num) => num,
};
match year.len() {
2 => {
let first = if num < 10 {
number_to_words(2000)
} else if num < 70 {
number_to_words(20)
} else {
number_to_words(19)
};
if first.is_none() { return None; }
let mut words = first.unwrap();
if num == 0 { return Some(words); }
let mut last = match number_to_words(num) {
None => { return None; },
Some(w) => w,
};
words.append(&mut last);
Some(words)
},
4 => {
if num % 1000 == 0 {
// Y000's
let first = number_to_words(num);
if first.is_none() { return None; }
return Some(first.unwrap());
} else if num % 100 == 0 {
// YY00's
let n = num / 100;
let first = number_to_words(n);
if first.is_none() { return None; }
let mut words = first.unwrap();
if n < 20 { words.push("hundred".to_string()); }
return Some(words);
} else {
// YY0X and YYXX.
let first = num / 100;
let second = num % 100;
if second < 10 {
if first % 10 == 0 {
// 2001, etc.
return number_to_words(num);
} else {
// 1901, etc.
let a = number_to_words(first);
if a.is_none() { return None; }
let mut words = a.unwrap();
if second < 10 {
words.push("o".to_string());
let b = number_to_words(second);
if b.is_none() { return None; }
let mut more = b.unwrap();
words.append(&mut more);
} else {
let b = number_to_words(second);
if b.is_none() { return None; }
let mut more = b.unwrap();
words.append(&mut more);
}
return Some(words);
}
} else {
let a = number_to_words(first);
if a.is_none() { return None; }
let mut words = a.unwrap();
let b = number_to_words(second);
if b.is_none() { return None; }
let mut more = b.unwrap();
words.append(&mut more);
return Some(words);
}
}
None // Should not reach
},
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_invalid_short_date() {
// TODO
}
#[test]
fn test_invalid_full_date() {
// TODO
}
#[test]
fn test_valid_short_date() {
// TODO
}
#[test]
fn test_valid_full_date() {
// 1900's
assert_eq!(words("january first nineteen eighty"),
date_to_words("1/1/1980"));
assert_eq!(words("october seventh nineteen eighty"),
date_to_words("10/7/80"));
assert_eq!(words("september ninth nineteen ninety nine"),
date_to_words("9/9/1999"));
assert_eq!(words("october thirty first nineteen o one"),
date_to_words("10/31/1901"));
// 2000's
assert_eq!(words("january first two thousand"),
date_to_words("1/1/2000"));
assert_eq!(words("december twelfth twenty twelve"),
date_to_words("12/12/2012"));
}
#[test]
fn test_invalid_month() {
assert_eq!(None, month_to_words("0"));
assert_eq!(None, month_to_words("00"));
assert_eq!(None, month_to_words("-1"));
assert_eq!(None, month_to_words("13"));
assert_eq!(None, month_to_words("111"));
assert_eq!(None, month_to_words("foo"));
}
#[test]
fn test_valid_month() {
assert_eq!(Some("january"), month_to_words("1"));
assert_eq!(Some("january"), month_to_words("01"));
assert_eq!(Some("july"), month_to_words("7"));
assert_eq!(Some("july"), month_to_words("07"));
assert_eq!(Some("october"), month_to_words("10"));
assert_eq!(Some("december"), month_to_words("12"));
}
#[test]
fn test_invalid_day() {
assert_eq!(None, day_to_words(""));
assert_eq!(None, day_to_words("0"));
assert_eq!(None, day_to_words("-1"));
assert_eq!(None, day_to_words("32"));
assert_eq!(None, day_to_words("100"));
assert_eq!(None, day_to_words("foo"));
}
#[test]
fn test_valid_day() {
assert_eq!(words("first"), day_to_words("1"));
assert_eq!(words("fifth"), day_to_words("5"));
assert_eq!(words("tenth"), day_to_words("10"));
assert_eq!(words("fifteenth"), day_to_words("15"));
assert_eq!(words("twentieth"), day_to_words("20"));
assert_eq!(words("twenty first"), day_to_words("21"));
assert_eq!(words("thirty first"), day_to_words("31"));
}
#[test]
fn test_wrong_digit_year() {
// Error
assert_eq!(None, year_to_words(""));
assert_eq!(None, year_to_words("0"));
assert_eq!(None, year_to_words("1"));
//assert_eq!(None, year_to_words("-1")); // TODO: Fix
assert_eq!(None, year_to_words("123"));
assert_eq!(None, year_to_words("12345"));
assert_eq!(None, year_to_words("foo"));
}
#[test]
fn test_two_digit_year() {
// 2000's (two digit)
assert_eq!(words("two thousand"), year_to_words("00"));
assert_eq!(words("two thousand one"), year_to_words("01"));
assert_eq!(words("two thousand four"), year_to_words("04"));
assert_eq!(words("twenty nineteen"), year_to_words("19"));
assert_eq!(words("twenty twenty"), year_to_words("20"));
assert_eq!(words("twenty fifty five"), year_to_words("55"));
assert_eq!(words("twenty sixty nine"), year_to_words("69"));
// 1900's (two digit)
assert_eq!(words("nineteen seventy"), year_to_words("70"));
assert_eq!(words("nineteen eighty five"), year_to_words("85"));
assert_eq!(words("nineteen ninety five"), year_to_words("95"));
assert_eq!(words("nineteen ninety nine"), year_to_words("99"));
}
#[test]
fn test_four_digit_year() {
// thousands (four digit years)
assert_eq!(words("one thousand"), year_to_words("1000"));
assert_eq!(words("two thousand"), year_to_words("2000"));
// hundreds (four digit years)
assert_eq!(words("eleven hundred"), year_to_words("1100"));
assert_eq!(words("eighteen hundred"), year_to_words("1800"));
assert_eq!(words("nineteen hundred"), year_to_words("1900"));
// ones digit (four digit years before 2000)
assert_eq!(words("eleven o one"), year_to_words("1101"));
assert_eq!(words("seventeen o nine"), year_to_words("1709"));
// ones digit (four digit years after 2000)
assert_eq!(words("two thousand one"), year_to_words("2001"));
assert_eq!(words("two thousand two"), year_to_words("2002"));
assert_eq!(words("four thousand five"), year_to_words("4005"));
// others
assert_eq!(words("nineteen eighty five"), year_to_words("1985"));
assert_eq!(words("nineteen ninety seven"), year_to_words("1997"));
assert_eq!(words("twenty ten"), year_to_words("2010"));
assert_eq!(words("twenty twelve"), year_to_words("2012"));
assert_eq!(words("twenty sixteen"), year_to_words("2016"));
assert_eq!(words("twenty twenty"), year_to_words("2020"));
}
fn words(words: &str) -> Option<Vec<String>> {
let list = words.split_whitespace().map(|w| w.to_string()).collect();
Some(list)
}
}
+139
View File
@@ -0,0 +1,139 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
use arpabet::Arpabet;
use speaker::Speaker;
use std::collections::HashMap;
use std::collections::HashSet;
pub type Word = String;
pub struct Dictionary {
/// Words that exist in this dictionary.
words: HashSet<Word>,
}
pub struct UniversalDictionary {
/// There is a single arpabet dictionary that works for all speakers.
arpabet_dictionary: Dictionary,
/// There are speaker-specific word dictionaries.
speaker_dictionaries: HashMap<Speaker, Dictionary>,
}
impl Dictionary {
pub fn new(words: HashSet<Word>) -> Dictionary {
Dictionary { words: words }
}
pub fn empty() -> Dictionary {
Dictionary { words: HashSet::new() }
}
/// Extract and return the internal word dictionary (a list of all
/// the words contained in the arpabet).
pub fn from_arpabet(arpabet: &Arpabet) -> Dictionary {
let mut words = HashSet::new();
for word in arpabet.keys() {
words.insert(word.to_string());
}
Dictionary::new(words)
}
/// Returns true if the dictionary contains the word.
pub fn contains(&self, word: &str) -> bool {
self.words.contains(word)
}
/// For testing only.
#[allow(dead_code)]
fn insert(mut self, word: &str) -> Self {
self.words.insert(word.to_string());
self
}
}
impl UniversalDictionary {
pub fn new() -> UniversalDictionary {
UniversalDictionary {
arpabet_dictionary: Dictionary::empty(),
speaker_dictionaries: HashMap::new(),
}
}
pub fn set_arpabet_dictionary(&mut self, arpabet_dictionary: Dictionary)
-> &mut Self {
self.arpabet_dictionary = arpabet_dictionary;
self
}
pub fn set_speaker_dictionary(&mut self,
speaker: Speaker,
dictionary: Dictionary)
-> &mut Self {
self.speaker_dictionaries.insert(speaker, dictionary);
self
}
/// Returns true if anyone can say the word.
pub fn contains(&self, word: &str) -> bool {
self.arpabet_dictionary.contains(word)
}
/// Returns true if the speaker can say the word.
pub fn contains_for(&self, speaker: &Speaker, word: &str) -> bool {
self.arpabet_dictionary.contains(word) ||
self.speaker_dictionaries.get(speaker)
.map(|dict| { dict.contains(word) })
.unwrap_or(false)
}
}
#[cfg(test)]
mod tests {
use speaker::Speaker;
use super::*;
#[test]
fn universal_dictionary_contains_for_speaker() {
let alice = Speaker::new("alice".to_string());
let bob = Speaker::new("bob".to_string());
let arpa = Dictionary::empty()
.insert("foo")
.insert("bar");
let alice_dict = Dictionary::empty()
.insert("111")
.insert("222");
let bob_dict = Dictionary::empty()
.insert("aaa")
.insert("bbb");
let mut uni = UniversalDictionary::new();
uni.set_arpabet_dictionary(arpa)
.set_speaker_dictionary(alice.clone(), alice_dict)
.set_speaker_dictionary(bob.clone(), bob_dict);
assert_eq!(true, uni.contains_for(&alice, "foo"));
assert_eq!(true, uni.contains_for(&alice, "bar"));
assert_eq!(false, uni.contains_for(&alice, "baz"));
assert_eq!(true, uni.contains_for(&alice, "111"));
assert_eq!(true, uni.contains_for(&alice, "222"));
assert_eq!(false, uni.contains_for(&alice, "aaa"));
assert_eq!(false, uni.contains_for(&alice, "bbb"));
assert_eq!(true, uni.contains_for(&bob, "foo"));
assert_eq!(false, uni.contains_for(&bob, "111"));
assert_eq!(true, uni.contains_for(&bob, "aaa"));
assert_eq!(false, uni.contains_for(&bob, "absent"));
let nobody = Speaker::new("nobody".to_string());
assert_eq!(true, uni.contains_for(&nobody, "foo"));
assert_eq!(false, uni.contains_for(&nobody, "111"));
assert_eq!(false, uni.contains_for(&nobody, "aaa"));
assert_eq!(false, uni.contains_for(&nobody, "absent"));
}
}
+12
View File
@@ -0,0 +1,12 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
pub mod abbr;
pub mod dates;
pub mod dictionary;
pub mod numbers;
pub mod ordinals;
pub mod parser;
pub mod phonemes;
pub mod syllables;
pub mod tokenizer;
pub mod tokens;
+292
View File
@@ -0,0 +1,292 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
// Inspired by http://stackoverflow.com/a/3299672
use std::usize;
use std::u64;
lazy_static! {
static ref FIRST_DIGITS: Vec<&'static str> = vec![
"zero",
"one",
"two",
"three",
"four",
"five",
"six",
"seven",
"eight",
"nine",
"ten",
"eleven",
"twelve",
"thirteen",
"fourteen",
"fifteen",
"sixteen",
"seventeen",
"eighteen",
"nineteen",
];
static ref TENS: Vec<&'static str> = vec![
"twenty",
"thirty",
"forty",
"fifty",
"sixty",
"seventy",
"eighty",
"ninety",
];
static ref LARGE: Vec<&'static str> = vec![
"thousand",
"million",
"billion",
"trillion",
"quadrillion",
// TODO: Overflow in current algorithm:
//"quintillion",
//"sextillion",
//"septillion",
//"octillion",
//"nonillion",
];
}
/// Convert a number to words.
pub fn number_to_words(number: i64) -> Option<Vec<String>> {
let mut words = Vec::new();
if number < 0 {
words.push("negative".to_string());
}
let unsigned = number.abs() as u64;
let success = if unsigned < 100 {
convert_nn(unsigned, &mut words)
} else if unsigned < 1000 {
convert_nnn(unsigned, &mut words)
} else {
convert_large(unsigned, &mut words)
};
if !success {
None
} else {
Some(words)
}
}
// TODO: This algorithm was copied from SO.
// Could be more elegant. Also more Rustful.
fn convert_nn(number: u64, words: &mut Vec<String>) -> bool {
if number < 20 {
let word = FIRST_DIGITS[number as usize].to_string();
words.push(word);
return true;
}
for (i, word) in TENS.iter().enumerate() {
let dval = 10 * i + 20;
if dval + 10 > number as usize {
if (number % 10) != 0 {
let first = word.to_string();
let second = FIRST_DIGITS[(number % 10) as usize].to_string();
words.push(first);
words.push(second);
return true;
}
words.push(word.to_string());
return true;
}
}
false // Should not be reached.
}
fn convert_nnn(number: u64, words: &mut Vec<String>) -> bool {
let rem = number / 100;
if rem > 0 {
words.push(FIRST_DIGITS[rem as usize].to_string());
words.push("hundred".to_string());
}
let md = number % 100;
if md > 0 {
if !convert_nn(md, words) {
return false;
}
}
true
}
fn convert_large(number: u64, words: &mut Vec<String>) -> bool {
// Extremly large numbers
if number > 999_999_999_999_999_999 {
return false;
}
if number < 1000 {
if !convert_nnn(number, words) {
return false;
}
return true;
}
// Iterate backwards, largest magnitudes first.
for (i, unit) in LARGE.iter().rev().enumerate() {
let i = LARGE.len() - i; // shadow
let mag = 1000u64.pow(i as u32);
if number < mag {
continue;
}
let quo = number / mag;
let rem = number - (quo * mag);
if !convert_nnn(quo, words) {
return false;
}
words.push(unit.to_string());
if rem > 0 {
if !convert_large(rem, words) {
return false;
}
}
return true;
}
false // Should not be reached.
}
#[cfg(test)]
mod tests {
use super::number_to_words;
#[test]
fn test_positive_integers() {
// Ones and teens
assert_eq!(w("zero"), number_to_words(0));
assert_eq!(w("one"), number_to_words(1));
assert_eq!(w("two"), number_to_words(2));
assert_eq!(w("three"), number_to_words(3));
assert_eq!(w("four"), number_to_words(4));
assert_eq!(w("five"), number_to_words(5));
assert_eq!(w("six"), number_to_words(6));
assert_eq!(w("seven"), number_to_words(7));
assert_eq!(w("eight"), number_to_words(8));
assert_eq!(w("nine"), number_to_words(9));
assert_eq!(w("ten"), number_to_words(10));
assert_eq!(w("eleven"), number_to_words(11));
assert_eq!(w("twelve"), number_to_words(12));
assert_eq!(w("thirteen"), number_to_words(13));
assert_eq!(w("fourteen"), number_to_words(14));
assert_eq!(w("fifteen"), number_to_words(15));
assert_eq!(w("sixteen"), number_to_words(16));
assert_eq!(w("seventeen"), number_to_words(17));
assert_eq!(w("eighteen"), number_to_words(18));
assert_eq!(w("nineteen"), number_to_words(19));
// Tens
assert_eq!(w("twenty"), number_to_words(20));
assert_eq!(w("thirty"), number_to_words(30));
assert_eq!(w("forty"), number_to_words(40));
assert_eq!(w("fifty"), number_to_words(50));
assert_eq!(w("sixty"), number_to_words(60));
assert_eq!(w("seventy"), number_to_words(70));
assert_eq!(w("eighty"), number_to_words(80));
assert_eq!(w("ninety"), number_to_words(90));
// Tens + Ones
assert_eq!(w("twenty one"), number_to_words(21));
assert_eq!(w("thirty five"), number_to_words(35));
assert_eq!(w("forty four"), number_to_words(44));
assert_eq!(w("fifty five"), number_to_words(55));
assert_eq!(w("sixty six"), number_to_words(66));
// Hundreds
assert_eq!(w("one hundred"), number_to_words(100));
assert_eq!(w("two hundred"), number_to_words(200));
assert_eq!(w("three hundred"), number_to_words(300));
assert_eq!(w("four hundred"), number_to_words(400));
assert_eq!(w("five hundred"), number_to_words(500));
assert_eq!(w("six hundred"), number_to_words(600));
assert_eq!(w("seven hundred"), number_to_words(700));
assert_eq!(w("eight hundred"), number_to_words(800));
assert_eq!(w("nine hundred"), number_to_words(900));
// Large
assert_eq!(w("one thousand"), number_to_words(1_000));
assert_eq!(w("one million"), number_to_words(1_000_000));
assert_eq!(w("one billion"), number_to_words(1_000_000_000));
assert_eq!(w("one trillion"), number_to_words(1_000_000_000_000));
// Large with small prefix
assert_eq!(w("nine thousand"), number_to_words(9_000));
assert_eq!(w("ten million"), number_to_words(10_000_000));
assert_eq!(w("sixty billion"), number_to_words(60_000_000_000));
assert_eq!(w("four hundred forty four trillion"),
number_to_words(444_000_000_000_000));
// Large, multiple units
assert_eq!(w("nine thousand one"), number_to_words(9_001));
assert_eq!(w(r#"one hundred twenty three trillion
four hundred fifty six billion
seven hundred eighty nine million
one hundred twenty three thousand
four hundred fifty six"#),
number_to_words(123_456_789_123_456));
// Bounds
assert_eq!(w(r#"nine hundred ninety nine quadrillion
nine hundred ninety nine trillion
nine hundred ninety nine billion
nine hundred ninety nine million
nine hundred ninety nine thousand
nine hundred ninety nine"#),
number_to_words(999_999_999_999_999_999));
assert_eq!(None, number_to_words(1_000_000_000_000_000_000));
}
#[test]
fn test_negative_integers() {
// Misc negative numbers. (Algorithm already tested for positive ints.)
assert_eq!(w("negative one"), number_to_words(-1));
assert_eq!(w("negative nine thousand one"), number_to_words(-9_001));
assert_eq!(w("negative four hundred forty four trillion"),
number_to_words(-444_000_000_000_000));
assert_eq!(w(r#"negative one hundred twenty three trillion
four hundred fifty six billion
seven hundred eighty nine million
one hundred twenty three thousand
four hundred fifty six"#),
number_to_words(-123_456_789_123_456));
// Bounds
assert_eq!(w("zero"), number_to_words(-0));
assert_eq!(w(r#"negative
nine hundred ninety nine quadrillion
nine hundred ninety nine trillion
nine hundred ninety nine billion
nine hundred ninety nine million
nine hundred ninety nine thousand
nine hundred ninety nine"#),
number_to_words(-999_999_999_999_999_999));
assert_eq!(None, number_to_words(-1_000_000_000_000_000_000));
}
// Test helper
fn w(words: &str) -> Option<Vec<String>> {
Some(words.split_whitespace().map(|s| s.to_string()).collect::<Vec<String>>())
}
}
+143
View File
@@ -0,0 +1,143 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
use lang::numbers::number_to_words;
use regex::Regex;
use std::collections::HashMap;
lazy_static! {
pub static ref ORDINAL_MAP: HashMap<&'static str, &'static str> = {
let mut o = HashMap::new();
o.insert("zero", "zeroth");
o.insert("one", "first");
o.insert("two", "second");
o.insert("three", "third");
o.insert("four", "fourth");
o.insert("five", "fifth");
o.insert("six", "sixth");
o.insert("seven", "seventh");
o.insert("eight", "eighth");
o.insert("nine", "ninth");
o.insert("ten", "tenth");
o.insert("eleven", "eleventh");
o.insert("twelve", "twelfth");
o.insert("thirteen", "thirteenth");
o.insert("fourteen", "fourteenth");
o.insert("fifteen", "fifteenth");
o.insert("sixteen", "sixteenth");
o.insert("seventeen", "seventeenth");
o.insert("eighteen", "eighteenth");
o.insert("nineteen", "nineteenth");
o.insert("twenty", "twentieth");
o.insert("thirty", "thirtieth");
o.insert("forty", "fortieth");
o.insert("fifty", "fiftieth");
o.insert("sixty", "sixtieth");
o.insert("seventy", "seventieth");
o.insert("eighty", "eightieth");
o.insert("ninety", "ninetieth");
o.insert("hundred", "hundredth");
o.insert("thousand", "thousandth");
o.insert("million", "millionth");
o.insert("billion", "billionth");
o.insert("trillion", "trillionth");
o.insert("quadrillion", "quadrillionth");
o
};
}
/// Convert an ordinal of the form '123rd' to 'one hundred twenty third'.
pub fn ordinal_to_words(ordinal: &str) -> Option<Vec<String>> {
lazy_static! {
// Yes, this allows '3st', etc. It's fine.
static ref SUFFIX: Regex = Regex::new(r"st|nd|rd|th").unwrap();
}
let digits = SUFFIX.replace_all(ordinal, "");
let translated = digits.parse::<i64>()
.ok()
.and_then(|num| number_to_words(num));
if translated.is_none() {
return None;
}
let mut words = translated.unwrap();
let last = match words.pop() {
None => { return None },
Some(last) => last,
};
// NB: Compiler can't infer String or str.
let ordinal = match ORDINAL_MAP.get::<str>(&last) {
None => { return None },
Some(ord) => ord,
};
words.push(ordinal.to_string());
Some(words)
}
#[cfg(test)]
mod tests {
use super::ordinal_to_words;
#[test]
fn test_ordinals() {
// Ones and teens
assert_eq!(w("zeroth"), ordinal_to_words("0th"));
assert_eq!(w("first"), ordinal_to_words("1st"));
assert_eq!(w("second"), ordinal_to_words("2nd"));
assert_eq!(w("third"), ordinal_to_words("3rd"));
assert_eq!(w("fourth"), ordinal_to_words("4th"));
assert_eq!(w("fifth"), ordinal_to_words("5th"));
assert_eq!(w("sixth"), ordinal_to_words("6th"));
assert_eq!(w("seventh"), ordinal_to_words("7th"));
assert_eq!(w("eighth"), ordinal_to_words("8th"));
assert_eq!(w("ninth"), ordinal_to_words("9th"));
assert_eq!(w("tenth"), ordinal_to_words("10th"));
assert_eq!(w("eleventh"), ordinal_to_words("11th"));
assert_eq!(w("twelfth"), ordinal_to_words("12th"));
assert_eq!(w("thirteenth"), ordinal_to_words("13th"));
assert_eq!(w("fourteenth"), ordinal_to_words("14th"));
assert_eq!(w("fifteenth"), ordinal_to_words("15th"));
assert_eq!(w("sixteenth"), ordinal_to_words("16th"));
assert_eq!(w("seventeenth"), ordinal_to_words("17th"));
assert_eq!(w("eighteenth"), ordinal_to_words("18th"));
assert_eq!(w("nineteenth"), ordinal_to_words("19th"));
// Tens
assert_eq!(w("twentieth"), ordinal_to_words("20th"));
assert_eq!(w("thirtieth"), ordinal_to_words("30th"));
assert_eq!(w("fortieth"), ordinal_to_words("40th"));
assert_eq!(w("fiftieth"), ordinal_to_words("50th"));
assert_eq!(w("sixtieth"), ordinal_to_words("60th"));
assert_eq!(w("seventieth"), ordinal_to_words("70th"));
assert_eq!(w("eightieth"), ordinal_to_words("80th"));
assert_eq!(w("ninetieth"), ordinal_to_words("90th"));
// Hundreds, and higher magnitudes
assert_eq!(w("one hundredth"), ordinal_to_words("100th"));
assert_eq!(w("one thousandth"), ordinal_to_words("1000th"));
assert_eq!(w("one millionth"), ordinal_to_words("1000000th"));
assert_eq!(w("one billionth"), ordinal_to_words("1000000000th"));
assert_eq!(w("one trillionth"), ordinal_to_words("1000000000000th"));
assert_eq!(w("one quadrillionth"), ordinal_to_words("1000000000000000th"));
// Compound
assert_eq!(w("twenty first"), ordinal_to_words("21st"));
assert_eq!(w("one hundred eleventh"), ordinal_to_words("111th"));
assert_eq!(w("two thousand five hundred third"),
ordinal_to_words("2503rd"));
assert_eq!(w("one quadrillion first"),
ordinal_to_words("1000000000000001st"));
}
// Test helper
fn w(words: &str) -> Option<Vec<String>> {
Some(words.split_whitespace().map(|s| s.to_string()).collect::<Vec<String>>())
}
}
+726
View File
@@ -0,0 +1,726 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
// Parser takes a raw input, passes it to the tokenizer, and then
// converts it into the final output that is fed to the synthesizer.
// (The synthesizer only has knowledge of how to handle filtered words.)
use lang::abbr::AbbreviationsMap;
use lang::dates::date_to_words;
use lang::dictionary::UniversalDictionary;
use lang::numbers::number_to_words;
use lang::ordinals::ordinal_to_words;
use lang::tokenizer::*;
use lang::tokens::*;
use std::sync::Arc;
use synthesis::tokens::*;
pub struct Parser {
tokenizer: Tokenizer,
dictionary: Arc<UniversalDictionary>,
abbreviations: Arc<AbbreviationsMap>,
}
impl Parser {
/// CTOR.
pub fn new(tokenizer: Tokenizer,
dictionary: Arc<UniversalDictionary>,
abbreviations: Arc<AbbreviationsMap>) -> Parser {
Parser {
tokenizer: tokenizer,
dictionary: dictionary,
abbreviations: abbreviations,
}
}
/// Tokenize and parse the sentence into usable output for the synthesizer.
pub fn parse(&self, raw_sentence: &str) -> Vec<SynthToken> {
let tokens = self.tokenizer.tokenize(raw_sentence);
// First round: attempt to swap out certain token types for others.
// TODO: Inefficient. Use a mutable linked list to modify in-place.
let mut processed_tokens = Vec::new();
for token in tokens {
match token {
Token::Hashtag { value: ref v } => {
let tag = v.value.replace("#", "");
if self.dictionary.contains(&tag.to_lowercase()) {
processed_tokens.push(Token::dictionary_word("hashtag".to_string()));
processed_tokens.push(Token::dictionary_word(tag));
continue;
}
match self.try_expand_camel_case(&tag) {
None => {
processed_tokens.push(Token::unknown(tag.to_string()));
},
Some(words) => {
processed_tokens.push(Token::dictionary_word("hashtag".to_string()));
for word in words {
processed_tokens.push(Token::dictionary_word(word.to_string()));
}
continue;
}
}
},
Token::Mention { value: ref v } => {
let name = v.value.replace("@", "");
if self.dictionary.contains(&name.to_lowercase()) {
processed_tokens.push(Token::dictionary_word(name));
continue;
}
match self.try_expand_camel_case(&name) {
None => {
processed_tokens.push(Token::unknown(name.to_string()));
},
Some(words) => {
for word in words {
processed_tokens.push(Token::dictionary_word(word.to_string()));
}
continue;
}
}
},
_ => {},
}
processed_tokens.push(token);
}
// Process monetary token structures.
// TODO: Inefficient.
let mut processed_tokens2 = Vec::new();
let mut currency : Option<CurrencySymbol> = None;
for token in processed_tokens {
match token {
Token::CurrencySymbol { value: ref v } => {
currency = Some(v.clone());
continue;
},
Token::Number { value: _ } => {},
Token::ScaleWord { value: _ } => {},
_ => {
if currency.is_some() {
processed_tokens2.push(currency.unwrap().to_word_token_plural());
currency = None;
}
},
}
processed_tokens2.push(token);
}
if currency.is_some() {
processed_tokens2.push(currency.unwrap().to_word_token_plural());
}
// Process all token types and construct output synthesis vector.
let mut synth_tokens = Vec::new();
for token in processed_tokens2 {
match token {
Token::CurrencySymbol { value: _v } => {}, // Skip (for now)
Token::Emoji { value: _v } => {}, // Skip (for now)
Token::Hashtag { value: _v } => {}, // Skip (for now)
Token::Mention { value: _v } => {}, // Skip (for now)
Token::Url { value: _v } => {}, // Skip (forever)
Token::Punctuation { value: v } => {
match v {
Punctuation::Comma
| Punctuation::Dash => {
synth_tokens.push(SynthToken::breath());
},
Punctuation::Period
| Punctuation::Question
| Punctuation::Exclamation => {
synth_tokens.push(SynthToken::full_stop());
},
Punctuation::Ellipsis
| Punctuation::Semicolon => {
synth_tokens.push(SynthToken::half_stop());
},
}
},
Token::MaybeTimeUnit { value: ref v } => {
synth_tokens.push(SynthToken::word(v.value.to_string()));
},
Token::ScaleWord { value : ref v } => {
synth_tokens.push(SynthToken::word(v.value.to_string()));
}
Token::DictionaryWord { value : ref v } => {
synth_tokens.push(SynthToken::word(v.value.to_string()));
}
Token::Date { value: ref v } => {
match date_to_words(&v.value) {
None => {
synth_tokens.push(SynthToken::filler(v.value.len()));
},
Some(date_words) => {
for word in date_words {
synth_tokens.push(SynthToken::word(word));
}
},
}
},
Token::Time { value: ref v } => {
// FIXME: Efficiency, cleanup
let mut valid = true;
let mut numbers = Vec::new();
let mut oclock = false;
let split = v.value.split(":").collect::<Vec<&str>>();
for (i, part) in split.iter().enumerate() {
match part.parse::<i64>() {
Ok(num) => {
if num == 0 {
if i == split.len() - 1 {
oclock = true;
}
continue;
}
numbers.push(num)
},
Err(_) => {
valid = false;
break;
},
};
}
if !valid {
synth_tokens.push(SynthToken::filler(v.value.len()));
continue;
}
let mut number_words = Vec::new();
for num in numbers {
match number_to_words(num) {
None => {
valid = false;
break;
},
Some(words) => {
for word in words { number_words.push(word); }
}
}
}
if valid {
for word in number_words {
synth_tokens.push(SynthToken::word(word));
}
if oclock {
synth_tokens.push(SynthToken::word("o'clock".to_string()));
}
} else {
synth_tokens.push(SynthToken::filler(v.value.len()));
}
},
// Integers (TODO: tokenize floats separately.)
Token::Number { value: ref v } => {
let num = match v.value.parse::<i64>() {
Err(_) => { continue; },
Ok(num) => num,
};
match number_to_words(num) {
None => { continue; },
Some(words) => {
for word in words { synth_tokens.push(SynthToken::word(word)); }
}
}
},
Token::Ordinal { value: ref v } => {
match ordinal_to_words(&v.value) {
None => { continue; },
Some(words) => {
for word in words { synth_tokens.push(SynthToken::word(word)); }
}
}
},
// Abbreviations are mapped to words (vs. initialisms)
Token::Abbreviation { value: ref v } => {
match self.abbreviations.get_words(&v.value) {
None => { continue; },
Some(words) => {
for word in words {
synth_tokens.push(SynthToken::word(word.to_string()));
}
}
}
},
// Initialisms are mapped to letters (vs. abbreviations)
Token::Initialism { value: ref v } => {
let letters : Vec<_> = v.value.split("").collect();
for letter in letters {
synth_tokens.push(SynthToken::word(letter.to_string()));
}
},
Token::CamelCaseString { value: ref v } => {
match self.try_expand_camel_case(&v.value) {
None => {
synth_tokens.push(SynthToken::filler(v.value.len()));
},
Some(words) => {
for word in words {
synth_tokens.push(SynthToken::word(word.to_string()));
}
}
}
},
Token::HyphenatedString { value: ref v } => {
match self.try_expand_hypenated(&v.value) {
None => {
synth_tokens.push(SynthToken::filler(v.value.len()));
},
Some(words) => {
for word in words {
synth_tokens.push(SynthToken::word(word.to_string()));
}
}
}
},
Token::Symbol { value : ref v } => {
match *v {
Symbol::Ampersand => synth_tokens.push(SynthToken::word("and".to_string())),
Symbol::AtSign => synth_tokens.push(SynthToken::word("at".to_string())),
Symbol::Percent => synth_tokens.push(SynthToken::word("percent".to_string())),
Symbol::Plus => synth_tokens.push(SynthToken::word("plus".to_string())),
Symbol::GreaterThan => {
synth_tokens.push(SynthToken::word("greater".to_string()));
synth_tokens.push(SynthToken::word("than".to_string()));
},
Symbol::LessThan => {
synth_tokens.push(SynthToken::word("less".to_string()));
synth_tokens.push(SynthToken::word("than".to_string()));
},
}
},
Token::Unknown { value : ref v } => {
synth_tokens.push(SynthToken::filler(v.value.len()))
},
};
}
info!(target: "parsing", "Final synth tokens = {:?}", synth_tokens);
synth_tokens
}
/// Attempt to expand a hyphenated non-dictionary word into a list
/// of dictionary words. If any of the words don't match, this fails.
fn try_expand_hypenated(&self, hyphenated: &str) -> Option<Vec<String>> {
let split : Vec<&str> = hyphenated.split("-").collect();
let mut words = Vec::new();
for word in split {
let lower = word.to_lowercase();
if !self.dictionary.contains(&lower) {
return None;
}
words.push(lower);
}
Some(words)
}
/// Attempt to expand a CamelCased non-dictionary word into a list
/// of dictionary words. If any of the words don't match, this fails.
fn try_expand_camel_case(&self, camel_case: &str) -> Option<Vec<String>> {
let mut candidate_words = Vec::new();
let mut word_buf = Vec::new();
// NB: This doesn't work for unicode. See Rust docs.
for ch in camel_case.chars() {
if ch.is_uppercase() && !word_buf.is_empty() {
let word = word_buf.join("");
candidate_words.push(word);
word_buf.clear();
}
word_buf.push(ch.to_string());
}
if !word_buf.is_empty() {
let word = word_buf.join("");
candidate_words.push(word);
}
info!(target: "parsing", "CamelCase breakdown: {:?}", candidate_words);
let mut words = Vec::new();
for word in candidate_words {
let lower = word.to_lowercase();
if !self.dictionary.contains(&lower) {
return None;
}
words.push(lower);
}
Some(words)
}
}
#[cfg(test)]
mod tests {
use lang::abbr::AbbreviationsMap;
use lang::dictionary::Dictionary;
use lang::dictionary::UniversalDictionary;
use lang::tokenizer::Tokenizer;
use std::collections::HashSet;
use std::sync::Arc;
use super::*;
use synthesis::tokens::*;
// TODO: Share this between the tokenizer and parser tests.
lazy_static! {
/// Word set to use in the tests.
static ref WORDS: HashSet<String> = {
let words = vec![
"a",
"at",
"atlanta",
"available",
"bad",
"bar",
"baz",
"be",
"can't",
"echelon",
"ending",
"five",
"foo",
"food",
"four",
"fox",
"friday",
"georgia",
"handle",
"hound",
"idea",
"in",
"it",
"it's",
"jack",
"join",
"jon",
"lantern",
"link",
"lose",
"lot",
"make",
"me",
"movement",
"never",
"o",
"of",
"one",
"people",
"place",
"quote",
"sign",
"snow",
"testing",
"that",
"the",
"thing",
"this",
"three",
"tickets",
"today",
"two",
"username",
"visit",
"will",
"would",
"you",
];
let mut hs = HashSet::new();
for w in words { hs.insert(w.to_string()); }
hs
};
}
#[test]
fn test_normalize_spaces() {
let p = make_parser();
let result = p.parse(" FOO BAR BAZ ");
let expected = vec![w("foo"), w("bar"), w("baz")];
assert_eq!(expected, result);
let result = p.parse("\nFOO\t\tBAR\r\rBAZ\r\n\t");
let expected = vec![w("foo"), w("bar"), w("baz")];
assert_eq!(expected, result);
}
#[test]
fn test_drop_quotes() {
let p = make_parser();
let result = p.parse("'It's a quote'");
let expected = vec![w("it's"), w("a"), w("quote")];
assert_eq!(expected, result);
let result = p.parse("\"Quote quote\"");
let expected = vec![w("quote"), w("quote")];
assert_eq!(expected, result);
// Drop smart quotes, periods.
let result = p.parse("A lot of people can’t handle it.");
let expected = vec![
w("a"), w("lot"), w("of"), w("people"), w("can't"), w("handle"), w("it"),
SynthToken::full_stop()
];
assert_eq!(expected, result);
}
#[test]
fn test_drop_urls() {
let p = make_parser();
let result = p.parse("Testing https://t.co/1A2b3cdEfG link");
let expected = vec![w("testing"), w("link")];
assert_eq!(expected, result);
let result = p.parse("Visit https://t.co/A1A1b0b0b0…");
let expected = vec![w("visit"), SynthToken::half_stop()];
assert_eq!(expected, result);
}
#[test]
fn test_symbols() {
let p = make_parser();
let result = p.parse("Fox & Hound");
let expected = vec![w("fox"), w("and"), w("hound")];
assert_eq!(expected, result);
let result = p.parse("thing @ place");
let expected = vec![w("thing"), w("at"), w("place")];
assert_eq!(expected, result);
let result = p.parse("me > you");
let expected = vec![w("me"), w("greater"), w("than"), w("you")];
assert_eq!(expected, result);
let result = p.parse("this < that");
let expected = vec![w("this"), w("less"), w("than"), w("that")];
assert_eq!(expected, result);
}
#[test]
fn test_handle_punctuation() {
let p = make_parser();
let result = p.parse("“It would be a bad idea.”");
let expected = vec![
w("it"), w("would"), w("be"), w("a"), w("bad"), w("idea"),
SynthToken::full_stop()
];
assert_eq!(expected, result);
let result = p.parse("One, two, three... four!");
let expected = vec![
w("one"), SynthToken::breath(),
w("two"), SynthToken::breath(),
w("three"), SynthToken::half_stop(),
w("four"), SynthToken::full_stop()
];
assert_eq!(expected, result);
}
#[test]
fn test_camel_case_expansion() {
let p = make_parser();
let result = p.parse("FooBar");
let expected = vec![w("foo"), w("bar")];
assert_eq!(expected, result);
let result = p.parse("fooBarBaz");
let expected = vec![w("foo"), w("bar"), w("baz")];
assert_eq!(expected, result);
// A partially unknown camel case
let result = p.parse("FooBarQuerty");
let expected = vec![f(12)];
assert_eq!(expected, result);
// An entirely unknown camel case
let result = p.parse("qwertyUiop");
let expected = vec![f(10)];
assert_eq!(expected, result);
}
#[test]
fn test_hyphenated_expansion() {
let p = make_parser();
let result = p.parse("never-ending");
let expected = vec![w("never"), w("ending")];
assert_eq!(expected, result);
let result = p.parse("jack-o-lantern");
let expected = vec![w("jack"), w("o"), w("lantern")];
assert_eq!(expected, result);
// A partially unknown hyphenation
let result = p.parse("jack-o-qwerty");
let expected = vec![f(13)];
assert_eq!(expected, result);
// An entirely unknown hyphenation
let result = p.parse("qwerty-uiop");
let expected = vec![f(11)];
assert_eq!(expected, result);
}
#[test]
fn test_twitter_usernames() {
let p = make_parser();
// Simple dictionary word
let result = p.parse("@echelon");
let expected = vec![w("echelon")];
assert_eq!(expected, result);
// For CamelCase words that exist
let result = p.parse("@JonSnow");
let expected = vec![w("jon"), w("snow")];
assert_eq!(expected, result);
// Unknown
let result = p.parse("@qwerty");
let expected = vec![f(6)];
assert_eq!(expected, result);
}
#[test]
fn test_twitter_hashtags() {
let p = make_parser();
// Simple dictionary word
let result = p.parse("#food");
let expected = vec![w("hashtag"), w("food")];
assert_eq!(expected, result);
// For CamelCase words that exist
let result = p.parse("#FooBarBaz");
let expected = vec![w("hashtag"), w("foo"), w("bar"), w("baz")];
assert_eq!(expected, result);
// Unknown
let result = p.parse("#qwerty");
let expected = vec!(f(6));
assert_eq!(expected, result);
}
#[test]
fn test_times() {
let p = make_parser();
let result = p.parse("5:00");
let expected = vec![w("five"), w("o'clock")];
assert_eq!(expected, result);
let result = p.parse("12:30");
let expected = vec![w("twelve"), w("thirty")];
assert_eq!(expected, result);
let result = p.parse("5pm");
let expected = vec![w("five"), w("pm")];
assert_eq!(expected, result);
}
#[test]
fn test_monetary() {
let p = make_parser();
let result = p.parse("$10");
let expected = vec![w("ten"), w("dollars")];
assert_eq!(expected, result);
let result = p.parse("$ 9 thousand");
let expected = vec![w("nine"), w("thousand"), w("dollars")];
assert_eq!(expected, result);
let result = p.parse("$10 million");
let expected = vec![w("ten"), w("million"), w("dollars")];
assert_eq!(expected, result);
let result = p.parse("£123 billion");
let expected = vec![
w("one"), w("hundred"), w("twenty"), w("three"), w("billion"), w("pounds")
];
assert_eq!(expected, result);
let result = p.parse("€999 trillion");
let expected = vec![
w("nine"), w("hundred"), w("ninety"), w("nine"), w("trillion"), w("euros")
];
assert_eq!(expected, result);
// With other symbols in the mix.
let result = p.parse("Make $10 million!");
let expected = vec![
w("make"), w("ten"), w("million"), w("dollars"), SynthToken::full_stop()
];
assert_eq!(expected, result);
let result = p.parse("Lose $100 million today!");
let expected = vec![
w("lose"), w("one"),w("hundred"), w("million"), w("dollars"), w("today"),
SynthToken::full_stop()
];
assert_eq!(expected, result);
}
#[test]
fn test_complex_examples() {
let p = make_parser();
let result = p.parse(r#"Will be in Atlanta, Georgia this Friday at 5:00pm.
Join the MOVEMENT! Tickets available at:
https://t.co/Q6APf0ZFYA… https://t.co/6WAyO9eQHN"#);
let expected = vec![
w("will"), w("be"), w("in"), w("atlanta"), SynthToken::breath(),
w("georgia"), w("this"), w("friday"), w("at"), w("five"), w("o'clock"),
w("pm."), // TODO: Fix. Needs to be a full stop.
w("join"), w("the"), w("movement"), SynthToken::full_stop(),
w("tickets"), w("available"), w("at"),
SynthToken::half_stop()
];
assert_eq!(expected, result);
// TODO
// Handle numerics
//assert_eq!("u.s. murders increased ten point eight percent in twenty fifteen",
// &p.parse(&s, "U.S. Murders Increased 10.8% in 2015"));
}
// Helper function.
fn make_dictionary() -> UniversalDictionary {
let dictionary = Dictionary::new(WORDS.clone());
let mut ud = UniversalDictionary::new();
ud.set_arpabet_dictionary(dictionary);
ud
}
// Helper function.
fn make_parser() -> Parser {
let dictionary = Arc::new(make_dictionary());
let abbreviations = Arc::new(AbbreviationsMap::empty());
let tokenizer = Tokenizer::new(dictionary.clone(),
abbreviations.clone());
Parser::new(tokenizer, dictionary.clone(), abbreviations.clone())
}
fn f(value: u16) -> SynthToken {
SynthToken::Filler { value: Filler { length: value } }
}
fn w(value: &str) -> SynthToken {
SynthToken::Word { value: Word { value: value.to_string() } }
}
}
+360
View File
@@ -0,0 +1,360 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
#[derive(Debug)]
pub enum Stress {
// 0
None,
// 1
Primary,
// 2
Secondary
}
#[derive(Debug)]
pub enum Phoneme {
// VOWELS
// Monophthongs
Aa { stress: Stress },
Ae { stress: Stress },
Ah { stress: Stress },
Ao { stress: Stress },
Ax { stress: Stress },
Eh { stress: Stress },
Ih { stress: Stress },
Iy { stress: Stress },
Uh { stress: Stress },
Uw { stress: Stress },
// Diphthongs
Aw { stress: Stress },
Ay { stress: Stress },
Ey { stress: Stress },
Ow { stress: Stress },
Oy { stress: Stress },
// R-Colored
Er { stress: Stress },
// CONSONANTS
// Stops
B,
D,
G,
K,
P,
T,
// Affricates
Ch,
Jh,
// Fricatives
Dh,
F,
Hh,
S,
Sh,
Th,
V,
Z,
Zh,
// Nasals
Em,
En,
Eng,
M,
N,
Ng,
// Liquids
Dx,
El,
L,
Mx,
R,
// Semivowels
Q,
W,
Y,
}
impl Phoneme {
pub fn from_str(string: &str) -> Option<Phoneme> {
let phone = match string {
// CONSONANTS
// Stops
"B" => Phoneme::B,
"D" => Phoneme::D,
"G" => Phoneme::G,
"K" => Phoneme::K,
"P" => Phoneme::P,
"T" => Phoneme::T,
// Affricates
"CH" => Phoneme::Ch,
"JH" => Phoneme::Jh,
// Fricatives
"DH" => Phoneme::Dh,
"F" => Phoneme::F,
"HH" => Phoneme::Hh,
"S" => Phoneme::S,
"SH" => Phoneme::Sh,
"TH" => Phoneme::Th,
"V" => Phoneme::V,
"Z" => Phoneme::Z,
"Zh" => Phoneme::Zh,
// Nasals
"EM" => Phoneme::Em,
"EN" => Phoneme::En,
"ENG" => Phoneme::Eng,
"M" => Phoneme::M,
"N" => Phoneme::N,
"NG" => Phoneme::Ng,
// Liquids
"DX" => Phoneme::Dx,
"EL" => Phoneme::El,
"L" => Phoneme::L,
"MX" => Phoneme::Mx,
"R" => Phoneme::R,
// Semivowels
"Q" => Phoneme::Q,
"W" => Phoneme::W,
"Y" => Phoneme::Y,
// VOWELS
// Monophthongs
"AA0" => Phoneme::Aa { stress: Stress::None },
"AA1" => Phoneme::Aa { stress: Stress::Primary },
"AA2" => Phoneme::Aa { stress: Stress::Secondary },
"AE0" => Phoneme::Ae { stress: Stress::None },
"AE1" => Phoneme::Ae { stress: Stress::Primary },
"AE2" => Phoneme::Ae { stress: Stress::Secondary },
"AH0" => Phoneme::Ah { stress: Stress::None },
"AH1" => Phoneme::Ah { stress: Stress::Primary },
"AH2" => Phoneme::Ah { stress: Stress::Secondary },
"AO0" => Phoneme::Ao { stress: Stress::None },
"AO1" => Phoneme::Ao { stress: Stress::Primary },
"AO2" => Phoneme::Ao { stress: Stress::Secondary },
"AX0" => Phoneme::Ax { stress: Stress::None },
"AX1" => Phoneme::Ax { stress: Stress::Primary },
"AX2" => Phoneme::Ax { stress: Stress::Secondary },
"EH0" => Phoneme::Eh { stress: Stress::None },
"EH1" => Phoneme::Eh { stress: Stress::Primary },
"EH2" => Phoneme::Eh { stress: Stress::Secondary },
"IH0" => Phoneme::Ih { stress: Stress::None },
"IH1" => Phoneme::Ih { stress: Stress::Primary },
"IH2" => Phoneme::Ih { stress: Stress::Secondary },
"IY0" => Phoneme::Iy { stress: Stress::None },
"IY1" => Phoneme::Iy { stress: Stress::Primary },
"IY2" => Phoneme::Iy { stress: Stress::Secondary },
"UH0" => Phoneme::Uh { stress: Stress::None },
"UH1" => Phoneme::Uh { stress: Stress::Primary },
"UH2" => Phoneme::Uh { stress: Stress::Secondary },
"UW0" => Phoneme::Uw { stress: Stress::None },
"UW1" => Phoneme::Uw { stress: Stress::Primary },
"UW2" => Phoneme::Uw { stress: Stress::Secondary },
// Diphthongs
"AW0" => Phoneme::Aw { stress: Stress::None },
"AW1" => Phoneme::Aw { stress: Stress::Primary },
"AW2" => Phoneme::Aw { stress: Stress::Secondary },
"AY0" => Phoneme::Ay { stress: Stress::None },
"AY1" => Phoneme::Ay { stress: Stress::Primary },
"AY2" => Phoneme::Ay { stress: Stress::Secondary },
"EY0" => Phoneme::Ey { stress: Stress::None },
"EY1" => Phoneme::Ey { stress: Stress::Primary },
"EY2" => Phoneme::Ey { stress: Stress::Secondary },
"OW0" => Phoneme::Ow { stress: Stress::None },
"OW1" => Phoneme::Ow { stress: Stress::Primary },
"OW2" => Phoneme::Ow { stress: Stress::Secondary },
"OY0" => Phoneme::Oy { stress: Stress::None },
"OY1" => Phoneme::Oy { stress: Stress::Primary },
"OY2" => Phoneme::Oy { stress: Stress::Secondary },
// R-Colored
"ER0" => Phoneme::Er { stress: Stress::None },
"ER1" => Phoneme::Er { stress: Stress::Primary },
"ER2" => Phoneme::Er { stress: Stress::Secondary },
_ => {
return None;
},
};
Some(phone)
}
pub fn as_str(&self) -> &'static str {
match *self {
// VOWELS
// Monophthongs
Phoneme::Aa { ref stress } => {
match *stress {
Stress::None => "AA0",
Stress::Primary => "AA1",
Stress::Secondary => "AA2",
}
},
Phoneme::Ae { ref stress } => {
match *stress {
Stress::None => "AE0",
Stress::Primary => "AE1",
Stress::Secondary => "AE2",
}
},
Phoneme::Ah { ref stress } => {
match *stress {
Stress::None => "AH0",
Stress::Primary => "AH1",
Stress::Secondary => "AH2",
}
},
Phoneme::Ao { ref stress } => {
match *stress {
Stress::None => "AO0",
Stress::Primary => "AO1",
Stress::Secondary => "AO2",
}
},
Phoneme::Ax { ref stress } => {
match *stress {
Stress::None => "AX0",
Stress::Primary => "AX1",
Stress::Secondary => "AX2",
}
},
Phoneme::Eh { ref stress } => {
match *stress {
Stress::None => "EH0",
Stress::Primary => "EH1",
Stress::Secondary => "EH2",
}
},
Phoneme::Ih { ref stress } => {
match *stress {
Stress::None => "IH0",
Stress::Primary => "IH1",
Stress::Secondary => "IH2",
}
},
Phoneme::Iy { ref stress } => {
match *stress {
Stress::None => "IY0",
Stress::Primary => "IY1",
Stress::Secondary => "IY2",
}
},
Phoneme::Uh { ref stress } => {
match *stress {
Stress::None => "UH0",
Stress::Primary => "UH1",
Stress::Secondary => "UH2",
}
},
Phoneme::Uw { ref stress } => {
match *stress {
Stress::None => "UW0",
Stress::Primary => "UW1",
Stress::Secondary => "UW2",
}
},
// Diphthongs
Phoneme::Aw { ref stress } => {
match *stress {
Stress::None => "AW0",
Stress::Primary => "AW1",
Stress::Secondary => "AW2",
}
},
Phoneme::Ay { ref stress } => {
match *stress {
Stress::None => "AY0",
Stress::Primary => "AY1",
Stress::Secondary => "AY2",
}
},
Phoneme::Ey { ref stress } => {
match *stress {
Stress::None => "EY0",
Stress::Primary => "EY1",
Stress::Secondary => "EY2",
}
},
Phoneme::Ow { ref stress } => {
match *stress {
Stress::None => "OW0",
Stress::Primary => "OW1",
Stress::Secondary => "OW2",
}
},
Phoneme::Oy { ref stress } => {
match *stress {
Stress::None => "OY0",
Stress::Primary => "OY1",
Stress::Secondary => "OY2",
}
},
// R-Colored
Phoneme::Er { ref stress } => {
match *stress {
Stress::None => "ER0",
Stress::Primary => "ER1",
Stress::Secondary => "ER2",
}
},
// CONSONANTS
// Stops
Phoneme::B => "B",
Phoneme::D => "D",
Phoneme::G => "G",
Phoneme::K => "K",
Phoneme::P => "P",
Phoneme::T => "T",
// Affricates
Phoneme::Ch => "CH",
Phoneme::Jh => "JH",
// Fricatives
Phoneme::Dh => "DH",
Phoneme::F => "F",
Phoneme::Hh => "HH",
Phoneme::S => "S",
Phoneme::Sh => "SH",
Phoneme::Th => "TH",
Phoneme::V => "V",
Phoneme::Z => "Z",
Phoneme::Zh => "ZH",
// Nasals
Phoneme::Em => "EM",
Phoneme::En => "EN",
Phoneme::Eng => "ENG",
Phoneme::M => "M",
Phoneme::N => "N",
Phoneme::Ng => "NG",
// Liquids
Phoneme::Dx => "DX",
Phoneme::El => "EL",
Phoneme::L => "L",
Phoneme::Mx => "MX",
Phoneme::R => "R",
// Semivowels
Phoneme::Q => "Q",
Phoneme::W => "W",
Phoneme::Y => "Y",
}
}
/// Whether the phoneme is a vowel.
pub fn is_vowel(&self) -> bool {
match *self {
// Monophthongs
Phoneme::Aa { stress: _ } |
Phoneme::Ae { stress: _ } |
Phoneme::Ah { stress: _ } |
Phoneme::Ao { stress: _ } |
Phoneme::Ax { stress: _ } |
Phoneme::Eh { stress: _ } |
Phoneme::Ih { stress: _ } |
Phoneme::Iy { stress: _ } |
Phoneme::Uh { stress: _ } |
Phoneme::Uw { stress: _ } |
// Diphthong
Phoneme::Aw { stress: _ } |
Phoneme::Ay { stress: _ } |
Phoneme::Ey { stress: _ } |
Phoneme::Ow { stress: _ } |
Phoneme::Oy { stress: _ } |
// R-Colored
Phoneme::Er { stress: _ } => true,
_ => false,
}
}
}
+579
View File
@@ -0,0 +1,579 @@
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
use lang::phonemes::Phoneme;
use lang::phonemes::Stress;
/*
Arpabet has 54 different units:
- Vowels:
- 10 monophthongs (AO, AA, IY, UW, EH, IH, UH, AH, AX, AE)
- 5 diphthongs (EY, AY, OW, AW, OY)
- 8 R-colored vowels (ER, AXR, EH R, UH R, AO R, AA R, IH|IY R, AW R)
- Consonants:
- 6 Stops (P, B, T, D, K, G)
- 2 Affricates (CH, JH)
- 9 Fricatives (F, V, TH, DH, S, Z, SH, ZH, HH)
- 6 Nasals (M, EM, N, EN, NG, ENG)
- 5 Liquids (L, EL, R, DX, MX)
- 3 Semivowels (Y, W, Q)
*/
pub type Polysyllable = Vec<Vec<String>>;
/// Traits on syllables, in case I decide to change the underlying
/// representation.
pub trait Polysyllabic {
/// Number of monophones in a polysyllable (each syllable has n-many
/// monophones).
fn monophone_count(&self) -> usize;
}
/// Break an arpabet string into syllables comprised of arpabet monophones.
pub fn arpabet_to_syllables(input: &Vec<String>) -> Option<Polysyllable> {
let mut phones = Vec::new();
for atom in input {
match Phoneme::from_str(atom) {
None => {
return None;
},
Some(phone) => { phones.push(phone); },
}
}
let mut syllables = Vec::new();
let mut syllable = Vec::new();
let mut building_syllable = false;
let mut syllable_has_vowel = false;
let mut consonants_after_vowel = 0; // Count consonants after vowel
for phone in phones {
if !building_syllable {
// Starting the very first syllable of the word.
building_syllable = true;
consonants_after_vowel = 0;
if phone.is_vowel() {
syllable_has_vowel = true;
}
syllable.push(phone);
continue;
}
if syllable_has_vowel {
if phone.is_vowel() {
// Must create a new syllable so vowels don't run together!
// Find out where to allocate any run of consonants from the
// previous (current) syllable.
if consonants_after_vowel > 0 {
let movement = consonant_allocation(&syllable,
consonants_after_vowel);
match movement {
ConsonantAlloc::MoveNone => {
syllables.push(syllable);
syllable = Vec::new();
},
ConsonantAlloc::MoveOne => {
let one = syllable.pop();
syllables.push(syllable);
syllable = Vec::new();
syllable.push(one.unwrap());
},
ConsonantAlloc::MoveTwo => {
let one = syllable.pop();
let two = syllable.pop();
syllables.push(syllable);
syllable = Vec::new();
syllable.push(two.unwrap());
syllable.push(one.unwrap());
},
}
syllable.push(phone);
syllable_has_vowel = true;
consonants_after_vowel = 0;
continue;
}
syllables.push(syllable);
syllable = Vec::new();
syllable.push(phone);
syllable_has_vowel = true;
consonants_after_vowel = 0;
continue;
} else {
consonants_after_vowel += 1;
}
}
if phone.is_vowel() {
// The first vowel of the first syllable.
syllable_has_vowel = true;
consonants_after_vowel = 0;
}
// Continue building current syllable.
syllable.push(phone);
}
if syllable.len() > 0 {
syllables.push(syllable);
}
// Convert back to string.
// FIXME: So inefficient! Linked lists would probably be best here.
let mut output_syllables = Vec::new();
for syllable in syllables {
let mut output_syllable = Vec::new();
for s in syllable {
output_syllable.push(s.as_str().to_string());
}
output_syllables.push(output_syllable);
}
info!(target: "syllable", "Syllables: {:?}", output_syllables);
Some(output_syllables)
}
// How many consonants to keep for a leading vowel.
enum ConsonantAlloc {
MoveNone,
MoveOne,
MoveTwo,
}
// Heuristics for how to handle the consonants between vowels.
// Determine which vowel they get allocated to.
#[inline]
fn consonant_allocation(syllable: &Vec<Phoneme>,
consonants_after_vowel: usize)
-> ConsonantAlloc {
match consonants_after_vowel {
4 => {
// There are four consonants between vowels.
// Of the four consonants, how many does the last vowel retain?
/*let mut movement = ConsonantAlloc::KeepTwo;
let first_consonant = syllable.get(syllable.len() - 4);
let second_consonant = syllable.get(syllable.len() - 3);
let third_consonant = syllable.get(syllable.len() - 2);
let fourth_consonant = syllable.get(syllable.len() - 1);
if first_consonant.is_some()
&& second_consonant.is_some()
&& third_consonant.is_some()
&& fourth_consonant.is_some() {
let first_consonant = first_consonant.unwrap();
let second_consonant = second_consonant.unwrap();
let third_consonant = third_consonant.unwrap();
let fourth_consonant = fourth_consonant.unwrap();
movement = match *first_consonant {
_ => ConsonantAlloc::KeepTwo,
};
}
movement*/
ConsonantAlloc::MoveTwo
},
3 => {
let first_consonant = syllable.get(syllable.len() - 3);
let second_consonant = syllable.get(syllable.len() - 2);
let third_consonant = syllable.get(syllable.len() - 1);
if first_consonant.is_some()
&& second_consonant.is_some()
&& third_consonant.is_some() {
let first_consonant = first_consonant.unwrap();
let second_consonant = second_consonant.unwrap();
let third_consonant = third_consonant.unwrap();
return match *first_consonant {
Phoneme::F => {
match *second_consonant {
Phoneme::T => ConsonantAlloc::MoveOne, // F T | ... (eg. soft_)
_ => ConsonantAlloc::MoveOne,
}
},
Phoneme::L => {
match *second_consonant {
Phoneme::S => ConsonantAlloc::MoveOne, // L S | ... (eg. false__)
_ => ConsonantAlloc::MoveOne,
}
},
Phoneme::M => {
match *second_consonant {
Phoneme::P => {
match *third_consonant {
Phoneme::L => ConsonantAlloc::MoveTwo, // M | P L ...
Phoneme::R => ConsonantAlloc::MoveTwo, // M | P R ...
Phoneme::T => ConsonantAlloc::MoveOne, // M P | T ...
_ => ConsonantAlloc::MoveOne,
}
},
_ => ConsonantAlloc::MoveOne,
}
},
Phoneme::Z => ConsonantAlloc::MoveTwo, // Z _ _ ...
_ => ConsonantAlloc::MoveOne,
};
}
ConsonantAlloc::MoveOne
},
2 => {
// Usually we allocate one consonant per vowel, which seems to work most
// of the time.
let last_vowel = syllable.get(syllable.len() - 3);
let first_consonant = syllable.get(syllable.len() - 2);
let second_consonant = syllable.get(syllable.len() - 1);
if last_vowel.is_some()
&& first_consonant.is_some()
&& second_consonant.is_some() {
let last_vowel = last_vowel.unwrap();
let first_consonant = first_consonant.unwrap();
let second_consonant = second_consonant.unwrap();
return match *first_consonant {
Phoneme::K => {
match *second_consonant {
Phoneme::Y => ConsonantAlloc::MoveTwo, // K Y ..
_ => ConsonantAlloc::MoveOne,
}
},
Phoneme::P => {
match *second_consonant {
Phoneme::L => ConsonantAlloc::MoveTwo, // P L ..
_ => ConsonantAlloc::MoveOne,
}
},
Phoneme::S => {
match *second_consonant {
Phoneme::T => {
match *last_vowel {
Phoneme::Ah { .. } => ConsonantAlloc::MoveOne, // (j)us | t..
Phoneme::Ih { .. } => ConsonantAlloc::MoveOne, // ..I S | T..
_ => ConsonantAlloc::MoveNone, // .. S T
}
},
//
_ => ConsonantAlloc::MoveOne,
}
}
Phoneme::T => {
match *second_consonant {
Phoneme::R => ConsonantAlloc::MoveTwo, // T R ..
_ => ConsonantAlloc::MoveOne,
}
},
_ => ConsonantAlloc::MoveOne,
};
}
ConsonantAlloc::MoveOne
},
1 => {
// Usually we want to assign the single consonant to the next vowel.
let previous_vowel = syllable.get(syllable.len() - 2);
let previous_consonant = syllable.get(syllable.len() - 1);
if previous_vowel.is_some() && previous_consonant.is_some() {
let previous_vowel = previous_vowel.unwrap();
let previous_consonant = previous_consonant.unwrap();
return match *previous_vowel {
Phoneme::Aa { .. } => {
match *previous_consonant {
Phoneme::L => ConsonantAlloc::MoveNone, // AA L .. ("All")
_ => ConsonantAlloc::MoveOne,
}
}
Phoneme::Ae { .. } => {
match *previous_consonant {
Phoneme::S => ConsonantAlloc::MoveNone, // AE S .. ("Ass")
_ => ConsonantAlloc::MoveOne,
}
},
Phoneme::Ah { stress: Stress::None } => {
match *previous_consonant {
Phoneme::N => ConsonantAlloc::MoveNone, // AH N ("Un")
_ => ConsonantAlloc::MoveOne,
}
},
Phoneme::Ah { stress: Stress::Secondary } => {
match *previous_consonant {
// AH2 L .. ("A(u)l", eg (n)ull-ify)
Phoneme::L => ConsonantAlloc::MoveNone,
Phoneme::N => ConsonantAlloc::MoveNone, // AH N ("Un")
_ => ConsonantAlloc::MoveOne,
}
},
Phoneme::Ao { .. } => {
match *previous_consonant {
Phoneme::R => ConsonantAlloc::MoveNone, // AO R .. ("Orr")
_ => ConsonantAlloc::MoveOne,
}
},
Phoneme::Ih { .. } => {
match *previous_consonant {
//Phoneme::K => ConsonantAlloc::MoveNone, // IH K .. ("Ick")
Phoneme::N => ConsonantAlloc::MoveNone, // IH N .. ("Inn")
Phoneme::S => ConsonantAlloc::MoveNone, // IH S .. ("Iss")
_ => ConsonantAlloc::MoveOne,
}
},
_ => ConsonantAlloc::MoveOne,
};
}
ConsonantAlloc::MoveOne
},
_ => ConsonantAlloc::MoveNone,
}
}
impl Polysyllabic for Polysyllable {
fn monophone_count(&self) -> usize {
let mut count = 0;
for syllable in self {
count += syllable.len();
}
count
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_easy_cases() {
// Test a bunch of random easy words from ARPABET.
// Reduction
assert_eq!(syl("R IH0|D AH1 K|SH AH0 N"), conv("R IH0 D AH1 K SH AH0 N"));
// Technological
//assert_eq!(syl("T EH2 K|N AH0|L AA1 JH|IH0|K AH0 L"),
// conv("T EH2 K N AH0 L AA1 JH IH0 K AH0 L"));
assert_eq!(syl("T EH2 K|N AH0|L AA1|JH IH0|K AH0 L"),
conv("T EH2 K N AH0 L AA1 JH IH0 K AH0 L"));
// Disappear
assert_eq!(syl("D IH2 S|AH0|P IH1 R"), conv("D IH2 S AH0 P IH1 R"));
// Disorientation
assert_eq!(syl("D IH0 S|AO2 R|IY0|AH0 N|T EY1|SH AH0 N"),
conv("D IH0 S AO2 R IY0 AH0 N T EY1 SH AH0 N"));
// Mountaineer
assert_eq!(syl("M AW1 N|T IH0 N|IH2 R"), conv("M AW1 N T IH0 N IH2 R"));
// Ridiculously
//assert_eq!(syl("R AH0|D IH1 K|Y AH0|L AH0 S|L IY0"),
// conv("R AH0 D IH1 K Y AH0 L AH0 S L IY0"));
assert_eq!(syl("R AH0|D IH1|K Y AH0|L AH0 S|L IY0"),
conv("R AH0 D IH1 K Y AH0 L AH0 S L IY0"));
// Because
assert_eq!(syl("B IH0|K AO1 Z"), conv("B IH0 K AO1 Z"));
// Passenger
assert_eq!(syl("P AE1 S|AH0 N|JH ER0"), conv("P AE1 S AH0 N JH ER0"));
// Wisconsin
assert_eq!(syl("W IH0 S|K AA1 N|S AH0 N"), conv("W IH0 S K AA1 N S AH0 N"));
// Kernaghan
//assert_eq!(syl("K ER0 N|AE1|G AH0 N"), conv("K ER0 N AE1 G AH0 N"));
// Diametrically
assert_eq!(syl("D AY2|AH0|M EH1|T R IH0|K AH0|L IY0"),
conv("D AY2 AH0 M EH1 T R IH0 K AH0 L IY0"));
// Diaper
assert_eq!(syl("D AY1|P ER0"), conv("D AY1 P ER0"));
// Number-one
assert_eq!(syl("N AH2 M|B ER0|W AH1 N"), conv("N AH2 M B ER0 W AH1 N"));
// Nullification
assert_eq!(syl("N AH2 L|AH0|F AH0|K EY1|SH AH0 N"),
conv("N AH2 L AH0 F AH0 K EY1 SH AH0 N"));
// Flabbergasted
assert_eq!(syl("F L AE1|B ER0|G AE2 S T|IH0 D"),
conv("F L AE1 B ER0 G AE2 S T IH0 D"));
// Molecules TODO
assert_eq!(syl("M AA1 L|AH0|K Y UW2 L Z"), conv("M AA1 L AH0 K Y UW2 L Z"));
// Nicaraguans
assert_eq!(syl("N IH2|K ER0|AA1 G|W AH0 N Z"),
conv("N IH2 K ER0 AA1 G W AH0 N Z"));
// Niceness
assert_eq!(syl("N AY1 S|N AH0 S"), conv("N AY1 S N AH0 S"));
// Repetition
//assert_eq!(syl("R EH2 P|AH0|T IH1 SH|AH0 N"),
// conv("R EH2 P AH0 T IH1 SH AH0 N"));
assert_eq!(syl("R EH2|P AH0|T IH1|SH AH0 N"),
conv("R EH2 P AH0 T IH1 SH AH0 N"));
// Repercussion
//assert_eq!(syl("R IY2|P ER0|K AH1 SH|AH0 N"),
// conv("R IY2 P ER0 K AH1 SH AH0 N"));
assert_eq!(syl("R IY2|P ER0|K AH1|SH AH0 N"),
conv("R IY2 P ER0 K AH1 SH AH0 N"));
// Limits
//assert_eq!(syl("L IH1 M|AH0 T S"), conv("L IH1 M AH0 T S"));
assert_eq!(syl("L IH1|M AH0 T S"), conv("L IH1 M AH0 T S"));
// Unimportant
assert_eq!(syl("AH0 N|IH0 M|P AO1 R|T AH0 N T"),
conv("AH0 N IH0 M P AO1 R T AH0 N T"));
// Unintended
assert_eq!(syl("AH2 N|IH0 N|T EH1 N|D IH0 D"),
conv("AH2 N IH0 N T EH1 N D IH0 D"));
// Velociraptor
assert_eq!(syl("V AH0|L AO1|S AH0|R AE2 P|T ER0"),
conv("V AH0 L AO1 S AH0 R AE2 P T ER0"));
/*
assert_eq!(syl(""), conv(""));
*/
}
#[test]
fn test_two_consonants() {
// Test words that rely on heuristics with two consonants between vowels
// Distance
assert_eq!(syl("D IH1 S|T AH0 N S"), conv("D IH1 S T AH0 N S"));
// Justice
assert_eq!(syl("JH AH1 S|T AH0 S"), conv("JH AH1 S T AH0 S"));
// Replenish
assert_eq!(syl("R IY0|P L EH1|N IH0 SH"), conv("R IY0 P L EH1 N IH0 SH"));
/*
assert_eq!(syl(""), conv(""));
*/
}
#[test]
fn test_three_consonants() {
// Test words with three consonants between vowels
// Employable (vs. empty)
assert_eq!(syl("EH0 M|P L OY1|AH0|B AH0 L"),
conv("EH0 M P L OY1 AH0 B AH0 L"));
// Empty (vs. employable)
assert_eq!(syl("EH1 M P|T IY0"), conv("EH1 M P T IY0"));
// Falsehoods
assert_eq!(syl("F AE1 L S|HH UH2 D Z"), conv("F AE1 L S HH UH2 D Z"));
// Software
assert_eq!(syl("S AO1 F T|W EH2 R"), conv("S AO1 F T W EH2 R"));
// Temperamental
assert_eq!(syl("T EH2 M|P R AH0|M EH1 N|T AH0 L"),
conv("T EH2 M P R AH0 M EH1 N T AH0 L"));
// Wisecracks
assert_eq!(syl("W AY1 Z|K R AE2 K S"), conv("W AY1 Z K R AE2 K S"));
/*
assert_eq!(syl(""), conv(""));
*/
}
#[test]
fn test_four_consonants() {
// TODO ENABLE - These don't work yet.
// Explain
assert_eq!(syl("IH0 K S|P L EY1 N"), conv("IH0 K S P L EY1 N"));
// Landscape
assert_eq!(syl("L AE1 N D|S K EY2 P"), conv("L AE1 N D S K EY2 P"));
// Redemption
assert_eq!(syl("R IH0|D EH1 M P|SH AH0 N"),
conv("R IH0 D EH1 M P SH AH0 N"));
/*
assert_eq!(syl(""), conv(""));
*/
}
//#[test]
fn test_difficult() {
// TODO ENABLE - These don't work yet.
// Talking
assert_eq!(syl("T AO1 K|IH0 NG"), conv("T AO1 K IH0 NG"));
// Archipelago
assert_eq!(syl("AA2 R K|AH0|P EH1 L|AH0|G OW2"),
conv("AA2 R K AH0 P EH1 L AH0 G OW2"));
}
#[test]
fn polysyllablic_monophone_count() {
fn syllable(input: Vec<Vec<&str>>) -> Polysyllable {
let mut out = Vec::new();
for syllable in input {
let new_syllable = syllable.iter().map(|s| s.to_string()).collect();
out.push(new_syllable);
}
out
}
let polyphone = syllable(vec![vec![]]);
assert_eq!(0, polyphone.monophone_count());
let polyphone = syllable(vec![vec!["A"]]);
assert_eq!(1, polyphone.monophone_count());
let polyphone = syllable(vec![vec!["A", "B", "C"], vec!["D"]]);
assert_eq!(4, polyphone.monophone_count());
let polyphone = syllable(vec![vec!["A", "B", "C"], vec!["D", "E"],
vec!["F", "G"]]);
assert_eq!(7, polyphone.monophone_count());
}
// Shorter for tests
fn conv(input: &str) -> Option<Vec<Vec<String>>> {
arpabet_to_syllables(&pho(input))
}
// Construct an arpabet polyphone
fn pho(words: &str) -> Vec<String> {
words.split_whitespace().map(|w| w.to_string()).collect()
}
// Construct syllables constructed of n-phones.
fn syl(words: &str) -> Option<Vec<Vec<String>>> {
let syllables: Vec<&str> = words.split("|").collect();
let mut syllables_of_phonemes = Vec::new();
for syllable in syllables {
let item = syllable.split_whitespace().map(|w| w.to_string()).collect();
syllables_of_phonemes.push(item);
}
Some(syllables_of_phonemes)
}
}
File diff suppressed because it is too large. Load diff
Loaded 100 of 125 files, more files were not shown because too many files have changed in this diff. Show more