Merge pull request #222 from storytold/bt-audio-recorder

voice conversion UI improvements
This commit is contained in:
Brandon Thomas authored and GitHub committed 2023-04-25 19:17:31 -07:00
commit 0dbe135781
13 files changed
+623 -214

No files matched your search

@@ -2,6 +2,7 @@ import { ApiConfig } from "../ApiConfig";
export interface UploadAudioRequest {
uuid_idempotency_token: string,
source?: string, // eg. "device", "file"
file: any,
}
@@ -33,6 +34,10 @@ export async function UploadAudio(request: UploadAudioRequest) : Promise<UploadA
formData.append('uuid_idempotency_token', request.uuid_idempotency_token);
formData.append('file', request.file);
if (request.source !== undefined) {
formData.append('source', request.source);
}
return fetch(endpoint, {
method: 'POST',
credentials: 'include',
+13 -1
View File
@@ -1,12 +1,20 @@
import { ModelInferenceJobStatus } from "../api/model_inference/GetModelInferenceJobStatus";
import { JobState, jobStateFromString } from "./JobStates";
// Type of inference job (specified by the frontend, not backend)
export enum FrontendInferenceJobType {
TextToSpeech,
VoiceConversion,
}
// NB: Many of these fields are optional despite the response payload containing them
// This is because we create temporary placeholder objects with just the token.
export class InferenceJob {
// PK
jobToken: string;
frontendJobType: FrontendInferenceJobType;
// Status
jobState: JobState;
maybeExtraStatusDescription: string | null;
@@ -26,6 +34,8 @@ export class InferenceJob {
constructor(
// PK
jobToken: string,
// Frontend state
frontendJobType: FrontendInferenceJobType,
// Status
status: string = 'unknown',
maybeExtraStatusDescription: string | null = null,
@@ -41,6 +51,7 @@ export class InferenceJob {
maybeResultPublicBucketMediaPath: string | undefined | null = null,
) {
this.jobToken = jobToken;
this.frontendJobType = frontendJobType;
this.jobState = jobStateFromString(status);
this.maybeExtraStatusDescription = maybeExtraStatusDescription;
this.attemptCount = attemptCount;
@@ -67,9 +78,10 @@ export class InferenceJob {
this.maybeResultPublicBucketMediaPath = maybeResultPublicBucketMediaPath;
}
static fromResponse(response: ModelInferenceJobStatus) : InferenceJob {
static fromResponse(response: ModelInferenceJobStatus, frontendJobType: FrontendInferenceJobType) : InferenceJob {
return new InferenceJob(
response.job_token,
frontendJobType,
response.status.status,
response.status.maybe_extra_status_description || null,
response.status.attempt_count || 0,
+30 -6
View File
@@ -44,7 +44,7 @@ import {
AVAILABLE_LANGUAGE_MAP,
ENGLISH_LANGUAGE,
} from "./_i18n/AvailableLanguageMap";
import { InferenceJob } from "@storyteller/components/src/jobs/InferenceJob";
import { FrontendInferenceJobType, InferenceJob } from "@storyteller/components/src/jobs/InferenceJob";
import { GetModelInferenceJobStatus, GetModelInferenceJobStatusIsOk } from "@storyteller/components/src/api/model_inference/GetModelInferenceJobStatus";
import { VoiceConversionModelUploadJob } from "@storyteller/components/src/jobs/VoiceConversionModelUploadJob";
import { VoiceConversionModelListItem } from "@storyteller/components/src/api/voice_conversion/ListVoiceConversionModels";
@@ -130,7 +130,6 @@ interface State {
isShowingBootstrapLanguageNotice: boolean;
// Jobs enqueued during this browser session.
inferenceJobs: Array<InferenceJob>;
ttsInferenceJobs: Array<TtsInferenceJob>;
w2lInferenceJobs: Array<W2lInferenceJob>;
ttsModelUploadJobs: Array<TtsModelUploadJob>;
@@ -138,6 +137,10 @@ interface State {
vocoderUploadJobs: Array<VocoderUploadJob>;
voiceConversionModelUploadJobs: Array<VoiceConversionModelUploadJob>;
// Generic inference jobs.
inferenceJobs: Array<InferenceJob>;
inferenceJobsByCategory: Map<FrontendInferenceJobType, Array<InferenceJob>>;
// Current text entered
textBuffer: string;
@@ -152,6 +155,13 @@ function newVocodes() {
return discord || twitter || alphaCookie;
}
function initInferenceJobsByCategoryMap() : Map<FrontendInferenceJobType, InferenceJob[]> {
let inferenceJobsByCategory = new Map();
inferenceJobsByCategory.set(FrontendInferenceJobType.TextToSpeech, []);
inferenceJobsByCategory.set(FrontendInferenceJobType.VoiceConversion, []);
return inferenceJobsByCategory;
}
function isMacOs() {
// Not on macs yet
// https://stackoverflow.com/a/38241481
@@ -176,6 +186,8 @@ class App extends React.Component<Props, State> {
let showPleaseFollowNotice = false;
let inferenceJobsByCategory = initInferenceJobsByCategoryMap();
this.state = {
enableAlpha: enableAlpha,
migrationMode: migrationMode,
@@ -194,7 +206,6 @@ class App extends React.Component<Props, State> {
isShowingPleaseFollowNotice: showPleaseFollowNotice,
isShowingBootstrapLanguageNotice: false,
inferenceJobs: [],
ttsInferenceJobs: [],
w2lInferenceJobs: [],
ttsModelUploadJobs: [],
@@ -202,6 +213,9 @@ class App extends React.Component<Props, State> {
vocoderUploadJobs: [],
voiceConversionModelUploadJobs: [],
inferenceJobs: [],
inferenceJobsByCategory: inferenceJobsByCategory,
textBuffer: "",
voiceConversionModels: [],
@@ -338,6 +352,8 @@ class App extends React.Component<Props, State> {
checkInferenceJob = async (jobToken: string) => {
const lookupResult = await GetModelInferenceJobStatus(jobToken);
let inferenceJobsByCategory = initInferenceJobsByCategoryMap();
if (GetModelInferenceJobStatusIsOk(lookupResult)) {
let updatedJobs: Array<InferenceJob> = [];
@@ -347,21 +363,28 @@ class App extends React.Component<Props, State> {
!jobStateCanChange(existingJob.jobState)
) {
updatedJobs.push(existingJob);
inferenceJobsByCategory.get(existingJob.frontendJobType)?.push(existingJob);
return;
}
let updatedJob = InferenceJob.fromResponse(lookupResult.state!);
let updatedJob = InferenceJob.fromResponse(
lookupResult.state!,
existingJob.frontendJobType
);
updatedJobs.push(updatedJob);
inferenceJobsByCategory.get(updatedJob.frontendJobType)?.push(existingJob);
});
this.setState({
inferenceJobs: updatedJobs,
inferenceJobsByCategory: inferenceJobsByCategory,
});
}
};
enqueueInferenceJob = (jobToken: string) => {
const newJob = new InferenceJob(jobToken);
enqueueInferenceJob = (jobToken: string, frontendJobType: FrontendInferenceJobType) => {
const newJob = new InferenceJob(jobToken, frontendJobType);
let inferenceJobs = this.state.inferenceJobs.concat([newJob]);
this.setState({
@@ -747,6 +770,7 @@ class App extends React.Component<Props, State> {
}
enqueueInferenceJob={this.enqueueInferenceJob}
inferenceJobs={this.state.inferenceJobs}
inferenceJobsByCategory={this.state.inferenceJobsByCategory}
enqueueTtsJob={this.enqueueTtsJob}
ttsInferenceJobs={this.state.ttsInferenceJobs}
enqueueW2lJob={this.enqueueW2lJob}
@@ -227,6 +227,14 @@ class Analytics {
Analytics.sendCategorizedEvent(this.VOICE_CONVERSION, 'open_primary_voice_select_menu');
}
static voiceConversionGenerate(modelToken: string) {
Analytics.sendCategorizedEvent(this.VOICE_CONVERSION, 'generate_vc', modelToken);
}
static voiceConversionClickDownload() {
Analytics.sendCategorizedEvent(this.VOICE_CONVERSION, 'click_download');
}
// ========== UI ==========
static uiTurnOnAnimations() {
@@ -68,7 +68,7 @@ import { NewsPage } from "./pages/news/NewsPage";
import { LandingPage } from "./pages/landing/LandingPage";
import { ChannelsPage } from "./pages/channels/Channels";
import { TrumpTtsPage } from "./pages/character/trump/TrumpTtsPage";
import { InferenceJob } from "@storyteller/components/src/jobs/InferenceJob";
import { FrontendInferenceJobType, InferenceJob } from "@storyteller/components/src/jobs/InferenceJob";
//import { LandingPage } from "./pages/landing/LandingPage";
import { VcModelListPage } from "./pages/vc/vc_model_list/VcModelListPage";
import { UploadVoiceConversionModel } from "./pages/upload/UploadVoiceConversionModel";
@@ -98,8 +98,9 @@ interface Props {
isShowingBootstrapLanguageNotice: boolean;
clearBootstrapLanguageNotice: () => void;
enqueueInferenceJob: (jobToken: string) => void;
enqueueInferenceJob: (jobToken: string, frontendInferenceJobType: FrontendInferenceJobType) => void;
inferenceJobs: Array<InferenceJob>;
inferenceJobsByCategory: Map<FrontendInferenceJobType, Array<InferenceJob>>;
enqueueTtsJob: (jobToken: string) => void;
ttsInferenceJobs: Array<TtsInferenceJob>;
@@ -478,15 +479,6 @@ class PageContainer extends React.Component<Props, State> {
<PatronPage sessionWrapper={this.props.sessionWrapper} />
</Route>
<Route path="/landing_temp">
<LandingPage
sessionWrapper={this.props.sessionWrapper}
sessionSubscriptionsWrapper={
this.props.sessionSubscriptionsWrapper
}
/>
</Route>
<Route path="/voice-conversion">
<VcModelListPage
sessionWrapper={this.props.sessionWrapper}
@@ -497,6 +489,9 @@ class PageContainer extends React.Component<Props, State> {
setVoiceConversionModels={this.props.setVoiceConversionModels}
maybeSelectedVoiceConversionModel={this.props.maybeSelectedVoiceConversionModel}
setMaybeSelectedVoiceConversionModel={this.props.setMaybeSelectedVoiceConversionModel}
enqueueInferenceJob={this.props.enqueueInferenceJob}
inferenceJobs={this.props.inferenceJobs}
inferenceJobsByCategory={this.props.inferenceJobsByCategory}
/>
</Route>
@@ -563,7 +558,7 @@ class PageContainer extends React.Component<Props, State> {
/>
</Route>
<Route path="/">
<Route path="/tts" exact={true}>
<TtsModelListPage
sessionWrapper={this.props.sessionWrapper}
sessionSubscriptionsWrapper={
@@ -588,6 +583,7 @@ class PageContainer extends React.Component<Props, State> {
}
enqueueInferenceJob={this.props.enqueueInferenceJob}
inferenceJobs={this.props.inferenceJobs}
inferenceJobsByCategory={this.props.inferenceJobsByCategory}
enqueueTtsJob={this.props.enqueueTtsJob}
ttsInferenceJobs={this.props.ttsInferenceJobs}
ttsModelUploadJobs={this.props.ttsModelUploadJobs}
@@ -623,6 +619,15 @@ class PageContainer extends React.Component<Props, State> {
}
/>
</Route>
<Route path="/" exact={true}>
<LandingPage
sessionWrapper={this.props.sessionWrapper}
sessionSubscriptionsWrapper={
this.props.sessionSubscriptionsWrapper
}
/>
</Route>
</Switch>
<FooterNav sessionWrapper={this.props.sessionWrapper} />
@@ -0,0 +1,256 @@
import React, { useEffect, useState } from "react";
import { t } from "i18next";
import { Trans } from "react-i18next";
import { Link } from "react-router-dom";
import { BucketConfig } from "@storyteller/components/src/api/BucketConfig";
import { JobState } from "@storyteller/components/src/jobs/JobStates";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faClock,
faHeadphonesSimple,
faDownload,
} from "@fortawesome/free-solid-svg-icons";
import { motion } from "framer-motion";
import { container, sessionItem } from "../../../data/animation";
import { SessionSubscriptionsWrapper } from "@storyteller/components/src/session/SessionSubscriptionsWrapper";
import { Analytics } from "../../../common/Analytics";
import { SessionTtsAudioPlayer } from "./SessionTtsAudioPlayer";
import { WebUrl } from "../../../common/WebUrl";
import {
GetPendingTtsJobCount,
GetPendingTtsJobCountIsOk,
GetPendingTtsJobCountSuccessResponse,
} from "@storyteller/components/src/api/tts/GetPendingTtsJobCount";
import { InferenceJob } from "@storyteller/components/src/jobs/InferenceJob";
interface Props {
inferenceJobs: Array<InferenceJob>;
sessionSubscriptionsWrapper: SessionSubscriptionsWrapper;
}
// TODO: This is duplicated in SessionTtsInferenceResultsList !
// Default to querying every 15 seconds, but make it configurable serverside
const DEFAULT_QUEUE_REFRESH_INTERVAL_MILLIS = 15000;
function SessionVoiceConversionResultsList(props: Props) {
const [pendingTtsJobs, setPendingTtsJobs] =
useState<GetPendingTtsJobCountSuccessResponse>({
success: true,
pending_job_count: 0,
cache_time: new Date(0), // NB: Epoch is used for vector clock's initial state
refresh_interval_millis: DEFAULT_QUEUE_REFRESH_INTERVAL_MILLIS,
});
useEffect(() => {
const fetch = async () => {
const response = await GetPendingTtsJobCount();
if (GetPendingTtsJobCountIsOk(response)) {
if (
response.cache_time.getTime() > pendingTtsJobs.cache_time.getTime()
) {
setPendingTtsJobs(response);
}
}
};
// TODO: We're having an outage and need to lower this.
//const interval = setInterval(async () => fetch(), 15000);
const refreshInterval = Math.max(DEFAULT_QUEUE_REFRESH_INTERVAL_MILLIS, pendingTtsJobs.refresh_interval_millis);
const interval = setInterval(async () => fetch(), refreshInterval);
fetch();
return () => clearInterval(interval);
}, [pendingTtsJobs]);
let results: Array<JSX.Element> = [];
// TODO(bt,2023-04-08): Clean this utter garbage duplication up.
// ============================= GENERIC INFERENCE =============================
props.inferenceJobs.forEach((job) => {
if (!job.maybeResultToken) {
let cssStyle = "alert alert-secondary mb-0";
let stateDescription = "Pending...";
switch (job.jobState) {
case JobState.PENDING:
case JobState.UNKNOWN:
stateDescription =
job.maybeExtraStatusDescription == null
? t("common.SessionTtsInferenceResults.progress.pending")
: job.maybeExtraStatusDescription;
break;
case JobState.STARTED:
cssStyle = "alert alert-success mb-0";
stateDescription =
job.maybeExtraStatusDescription == null
? t("common.SessionTtsInferenceResults.progress.started")
: job.maybeExtraStatusDescription;
break;
case JobState.ATTEMPT_FAILED:
cssStyle = "alert alert-danger mb-0";
stateDescription = `Failed ${job.attemptCount} attempt(s). Will retry...`;
break;
case JobState.COMPLETE_FAILURE:
case JobState.DEAD:
cssStyle = "alert alert-danger mb-0";
// TODO(bt,2023-01-23): Translate when I can test it
stateDescription = t(
"common.SessionTtsInferenceResults.progress.dead"
);
break;
case JobState.COMPLETE_SUCCESS:
cssStyle = "message is-success mb-0";
// Not sure why we're here instead of other branch!
stateDescription = t(
"common.SessionTtsInferenceResults.progress.success"
);
break;
}
results.push(
<div key={job.jobToken}>
<div>
<div>
<motion.div className={cssStyle} variants={sessionItem}>
{stateDescription}
</motion.div>
</div>
</div>
</div>
);
} else {
let audioLink = new BucketConfig().getGcsUrl(
job.maybeResultPublicBucketMediaPath
);
//let ttsPermalink = `/tts/result/${job.maybeResultToken}`;
let wavesurfers = <SessionTtsAudioPlayer filename={audioLink} />;
let audioDownloadFilename = `fakeyou-${job.jobToken}.wav`;
results.push(
<div key={job.jobToken}>
{/*<div className="message-header">
<p>{job.title}</p>
<button className="delete" aria-label="delete"></button>
</div>*/}
<div>
<motion.div
className="panel panel-tts-results p-4 gap-3 d-flex flex-column"
variants={sessionItem}
>
<div>
<h5 className="mb-2">{job.maybeModelTitle}</h5>
<p>{job.maybeRawInferenceText}</p>
</div>
{/* <audio
className="w-100"
controls
src={audioLink}
onClick={() => {
Analytics.ttsClickResultInlinePlay();
}}
>
Your browser does not support the
<code>audio</code> element.
</audio> */}
{wavesurfers}
<div className="mt-2">
{/*<Link
to={ttsPermalink}
onClick={() => {
Analytics.ttsClickResultLink();
}}
className="fw-semibold"
>
<FontAwesomeIcon icon={faLink} className="me-2" />
{t("common.SessionTtsInferenceResults.result.shareDownload")}
</Link>*/}
<a
className=" btn btn-primary w-100 mt-4"
href={audioLink}
onClick={() => {
Analytics.voiceConversionClickDownload();
}}
download={audioDownloadFilename}
>
<FontAwesomeIcon icon={faDownload} className="me-2" />
Download File{" "}
</a>
</div>
</motion.div>
</div>
</div>
);
}
});
let noResultsSection = (
<div className="panel panel-inner text-center p-5 rounded-5 h-100">
<div className="d-flex flex-column opacity-75 h-100 justify-content-center">
<FontAwesomeIcon icon={faHeadphonesSimple} className="fs-3 mb-3" />
<h5 className="fw-semibold">
{t("common.SessionTtsInferenceResults.noResults.title")}
</h5>
<p>{t("common.SessionTtsInferenceResults.noResults.subtitle")}</p>
</div>
</div>
);
if (results.length === 0) {
return <>{noResultsSection}</>;
}
let upgradeNotice = <></>;
// Ask non-premium users to upgrade
if (
results.length !== 0 &&
!props.sessionSubscriptionsWrapper.hasPaidFeatures()
) {
upgradeNotice = (
<div className="d-flex flex-column gap-3 sticky-top zi-2">
<motion.div
className="alert alert-warning alert-cta mb-0"
variants={sessionItem}
>
<FontAwesomeIcon icon={faClock} className="me-2" />
<Trans i18nKey="common.SessionTtsInferenceResults.premium.ad">
Don't want to wait? Step to the back of the line with a{" "}
<Link
to={WebUrl.pricingPageWithReferer("nowait")}
onClick={() => {
Analytics.ttsTooSlowUpgradePremium();
}}
className="alert-link fw-semibold"
>
FakeYou membership.
</Link>
</Trans>{" "}
(TTS Queued:{" "}
<span className="text-red">~{pendingTtsJobs.pending_job_count}</span>)
</motion.div>
</div>
);
}
// Users have requested reverse chronological results
results.reverse();
return (
<motion.div initial="hidden" animate="visible" variants={container}>
<div>
<div className="d-flex flex-column gap-3">
{upgradeNotice}
<div className="d-flex flex-column gap-3">{results}</div>
</div>
</div>
</motion.div>
);
}
export { SessionVoiceConversionResultsList };
@@ -127,7 +127,11 @@ function FooterNav(props: Props) {
<div className="py-2 col-12 col-lg-3 d-flex flex-column gap-2 gap-lg-3 align-items-center align-items-lg-start">
<p className="fw-bold">AI Tools</p>
<li>
<Link to="/">Text to Speech</Link>
<Link to="/tts">Text to Speech</Link>
</li>
<li>
<Link to="/voice-conversion">Voice Conversion</Link>
</li>
<li>
@@ -38,8 +38,7 @@ import "tippy.js/dist/tippy.css";
import "tippy.js/animations/shift-away.css";
import { ThirdPartyLinks } from "@storyteller/components/src/constants/ThirdPartyLinks";
import { Analytics } from "../../../common/Analytics";
import { faWaveformLines } from "@fortawesome/pro-solid-svg-icons";
import { faTvRetro } from "@fortawesome/pro-regular-svg-icons";
import { faMicrophoneLines, faWaveformLines } from "@fortawesome/pro-solid-svg-icons";
// TODO: This is duplicated in SessionTtsInferenceResultsList !
// Default to querying every 15 seconds, but make it configurable serverside
@@ -357,7 +356,7 @@ function TopNav(props: Props) {
</Link>
</li>
<li data-bs-toggle="offcanvas" className="nav-item">
{/*<li data-bs-toggle="offcanvas" className="nav-item">
<Link
to={WebUrl.newsPage()}
onClick={() => {
@@ -368,7 +367,7 @@ function TopNav(props: Props) {
<FontAwesomeIcon icon={faTvRetro} className="me-2" />
{t("nav.TopNav.main.newsLink")}
</Link>
</li>
</li>*/}
<li className="nav-item dropdown">
{/* TODO(echelon): Fix the build warnings about href not being accessible. */}
@@ -388,7 +387,7 @@ function TopNav(props: Props) {
aria-labelledby="offcanvasNavbarLgDropdown-tts"
>
<li data-bs-toggle="offcanvas">
<Link className="dropdown-item" to="/">
<Link className="dropdown-item" to="/tts">
<FontAwesomeIcon
icon={faWaveformLines}
className="me-2"
@@ -396,7 +395,6 @@ function TopNav(props: Props) {
{t("nav.TopNav.main.ttsOption")}
</Link>
</li>
{/* TODO(bt, 2023-01-11): Not ready to launch voice conversion
<li data-bs-toggle="offcanvas">
<Link className="dropdown-item" to="/voice-conversion">
<FontAwesomeIcon
@@ -407,7 +405,6 @@ function TopNav(props: Props) {
Voice Conversion
</Link>
</li>
*/}
<li data-bs-toggle="offcanvas">
<Link className="dropdown-item" to="/video">
<FontAwesomeIcon icon={faVideo} className="me-2" />
@@ -9,11 +9,6 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faUsers,
faStar,
faShapes,
faMicrophoneAlt,
faPaintBrush,
faPersonRunning,
faMusic,
faArrowRight,
faUser,
} from "@fortawesome/free-solid-svg-icons";
@@ -109,14 +104,16 @@ function LandingPage(props: Props) {
className="display-4 fw-bold lh-2 mb-3 zi-2"
variants={item}
>
Text to Speech,
AI Music,
<br />
Voice, and AI Tools
Text to Speech,
<br />
and Creator Tools
</motion.h1>
</div>
<div>
<motion.p className="lead mw-lead mb-3 px-2" variants={item}>
Use FakeYou deep fake technology to say stuff with your favorite
Use FakeYou deep fake technology to make movies and music with your favorite
characters.
</motion.p>
</div>
@@ -159,7 +156,7 @@ function LandingPage(props: Props) {
</div>
<div className="col-12 col-md-6 text-center">
<Link
to="/channels"
to="/voice-conversion"
className="w-100 d-flex flex-column align-items-center"
>
<div className="panel p-4 mt-5 mt-lg-0 panel-select">
@@ -168,7 +165,7 @@ function LandingPage(props: Props) {
src="/mascot/VC-img.webp"
alt="Voice Conversion"
/>
<h2 className="fw-bold text-white">Voice Conversion</h2>
<h2 className="fw-bold text-white">Music &amp; Voice Conversion</h2>
<h6 className="fw-normal opacity-75 text-white">
Speak as your favorite characters
</h6>
@@ -203,6 +200,7 @@ function LandingPage(props: Props) {
</div>
</motion.div>
{/*
<motion.div className="container-panel py-5" variants={panel}>
<div className="d-flex flex-column align-items-center text-center"></div>
<div className="d-flex justify-content-center">
@@ -255,6 +253,7 @@ function LandingPage(props: Props) {
</div>
</div>
</motion.div>
*/}
<motion.div className="container-panel my-5 py-5" variants={panel}>
<div className="panel p-3 p-lg-4 d-flex flex-column align-items-center justify-content-center">
@@ -7,7 +7,7 @@ import { SessionTtsModelUploadResultList } from "../../../_common/SessionTtsMode
import { SessionW2lInferenceResultList } from "../../../_common/SessionW2lInferenceResultsList";
import { SessionW2lTemplateUploadResultList } from "../../../_common/SessionW2lTemplateUploadResultsList";
import { SessionWrapper } from "@storyteller/components/src/session/SessionWrapper";
import { InferenceJob } from "@storyteller/components/src/jobs/InferenceJob";
import { FrontendInferenceJobType, InferenceJob } from "@storyteller/components/src/jobs/InferenceJob";
import { TtsInferenceJob } from "@storyteller/components/src/jobs/TtsInferenceJobs";
import { TtsModelUploadJob } from "@storyteller/components/src/jobs/TtsModelUploadJobs";
import { W2lInferenceJob } from "@storyteller/components/src/jobs/W2lInferenceJobs";
@@ -92,8 +92,9 @@ interface Props {
isShowingBootstrapLanguageNotice: boolean;
clearBootstrapLanguageNotice: () => void;
enqueueInferenceJob: (jobToken: string) => void;
enqueueInferenceJob: (jobToken: string, frontendInferenceJobType: FrontendInferenceJobType) => void;
inferenceJobs: Array<InferenceJob>;
inferenceJobsByCategory: Map<FrontendInferenceJobType, Array<InferenceJob>>;
enqueueTtsJob: (jobToken: string) => void;
ttsInferenceJobs: Array<TtsInferenceJob>;
@@ -299,7 +300,7 @@ function TtsModelListPage(props: Props) {
setMaybeTtsError(undefined);
if (response.inference_job_token_type === "generic") {
props.enqueueInferenceJob(response.inference_job_token);
props.enqueueInferenceJob(response.inference_job_token, FrontendInferenceJobType.TextToSpeech);
} else {
props.enqueueTtsJob(response.inference_job_token);
}
@@ -609,7 +610,7 @@ function TtsModelListPage(props: Props) {
</h4>
<div className="d-flex flex-column gap-3 session-tts-section">
<SessionTtsInferenceResultList
inferenceJobs={props.inferenceJobs}
inferenceJobs={props.inferenceJobsByCategory.get(FrontendInferenceJobType.TextToSpeech)!}
ttsInferenceJobs={props.ttsInferenceJobs}
sessionSubscriptionsWrapper={
props.sessionSubscriptionsWrapper
@@ -1,29 +1,25 @@
import React, { useState, useEffect, useCallback } from "react";
import { SessionWrapper } from "@storyteller/components/src/session/SessionWrapper";
import { t } from "i18next";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { motion } from "framer-motion";
import { v4 as uuidv4 } from "uuid";
import { container, panel } from "../../../../../data/animation";
import { SessionSubscriptionsWrapper } from "@storyteller/components/src/session/SessionSubscriptionsWrapper";
import { VcPageHero } from "./components/VcPageHero";
import Select, { createFilter } from "react-select";
import { SearchFieldClass } from "../../tts/tts_model_list/search/components/SearchFieldClass";
import {
faBarsStaggered,
faFiles,
faHeadphones,
faMicrophone,
faRightLeft,
faTrash,
} from "@fortawesome/pro-solid-svg-icons";
import UploadComponent from "./components/UploadComponent";
import RecordComponent from "./components/RecordComponent";
import { usePrefixedDocumentTitle } from "../../../../../common/UsePrefixedDocumentTitle";
import { ListVoiceConversionModels, VoiceConversionModelListItem } from "@storyteller/components/src/api/voice_conversion/ListVoiceConversionModels";
import { VcModelListSearch } from "./components/VcModelListSearchComponent";
import { EnqueueVoiceConversion, EnqueueVoiceConversionRequest } from "@storyteller/components/src/api/voice_conversion/EnqueueVoiceConversion";
import { EnqueueRemoteDownloadIsOk } from "@storyteller/components/src/api/remote_downloads/EnqueueRemoteDownload";
import { EnqueueVoiceConversion, EnqueueVoiceConversionIsSuccess, EnqueueVoiceConversionRequest } from "@storyteller/components/src/api/voice_conversion/EnqueueVoiceConversion";
import { Analytics } from "../../../../../common/Analytics";
import { FrontendInferenceJobType, InferenceJob } from "@storyteller/components/src/jobs/InferenceJob";
import { SessionVoiceConversionResultsList } from "../../../_common/SessionVoiceConversionResultsList";
interface Props {
sessionWrapper: SessionWrapper;
@@ -34,17 +30,25 @@ interface Props {
maybeSelectedVoiceConversionModel?: VoiceConversionModelListItem;
setMaybeSelectedVoiceConversionModel: (maybeSelectedVoiceConversionModel: VoiceConversionModelListItem) => void;
enqueueInferenceJob: (jobToken: string, frontendInferenceJobType: FrontendInferenceJobType) => void;
inferenceJobs: Array<InferenceJob>;
inferenceJobsByCategory: Map<FrontendInferenceJobType, Array<InferenceJob>>;
}
function VcModelListPage(props: Props) {
const [loading, setLoading] = useState(false);
usePrefixedDocumentTitle("Voice Conversion");
const [canConvert, setCanConvert] = useState(false);
const [mediaUploadToken, setMediaUploadToken] = useState<string|undefined>(undefined);
// Auto generated
const [idempotencyToken, setIdempotencyToken] = useState(uuidv4());
const [convertIdempotencyToken, setConvertIdempotencyToken] = useState(uuidv4());
usePrefixedDocumentTitle("Voice Conversion");
// NB: Something of a UI hack here.
// The 3rd party microphone component doesn't let you clear it, so we emulate form clearing
// with this variable.
const [formIsCleared, setFormIsCleared] = useState(false);
let {
setVoiceConversionModels,
@@ -53,10 +57,10 @@ function VcModelListPage(props: Props) {
setMaybeSelectedVoiceConversionModel,
} = props;
const ttsModelsLoaded = voiceConversionModels.length > 0;
const vcModelsLoaded = voiceConversionModels.length > 0;
const listModels = useCallback(async () => {
if (ttsModelsLoaded) {
if (vcModelsLoaded) {
return; // Already queried.
}
const models = await ListVoiceConversionModels();
@@ -77,36 +81,20 @@ function VcModelListPage(props: Props) {
setVoiceConversionModels,
maybeSelectedVoiceConversionModel,
setMaybeSelectedVoiceConversionModel,
ttsModelsLoaded,
vcModelsLoaded,
]);
const handleLoading = useCallback(() => {
setLoading(true);
setTimeout(() => {
setLoading(false);
}, 2000);
}, []);
useEffect(() => {
listModels();
const timeout = setTimeout(() => {
setLoading(false);
}, 2000);
return () => clearTimeout(timeout);
}, [
handleLoading,
listModels,
]);
}, [listModels]);
const handleClearClick = (ev: React.FormEvent<HTMLButtonElement>) => {
ev.preventDefault();
return false;
};
const changeConvertIdempotencyToken = () => {
setConvertIdempotencyToken(uuidv4());
}
const interceptModelChange = (maybeSelectedVoiceConversionModel: VoiceConversionModelListItem) => {
if (maybeSelectedVoiceConversionModel !== props.maybeSelectedVoiceConversionModel) {
setIdempotencyToken(uuidv4());
changeConvertIdempotencyToken();
}
props.setMaybeSelectedVoiceConversionModel(maybeSelectedVoiceConversionModel);
}
@@ -117,52 +105,33 @@ function VcModelListPage(props: Props) {
}
let request : EnqueueVoiceConversionRequest = {
uuid_idempotency_token: idempotencyToken,
uuid_idempotency_token: convertIdempotencyToken,
voice_conversion_model_token: props.maybeSelectedVoiceConversionModel.token,
source_media_upload_token: mediaUploadToken,
};
Analytics.voiceConversionGenerate(props.maybeSelectedVoiceConversionModel.token);
let result = await EnqueueVoiceConversion(request);
if (EnqueueRemoteDownloadIsOk(result)) {
console.log("successful enqueue");
if (EnqueueVoiceConversionIsSuccess(result)) {
props.enqueueInferenceJob(result.inference_job_token, FrontendInferenceJobType.VoiceConversion);
}
};
const handleFormSubmit = async (ev: React.FormEvent<HTMLFormElement>) => {
ev.preventDefault();
};
const loading = false; // TODO: Remove.
const speakButtonClass = loading
? "btn btn-primary w-100 disabled"
: "btn btn-primary w-100";
const canBeginConversion = mediaUploadToken !== undefined && props.maybeSelectedVoiceConversionModel !== undefined;
console.log('canBeginConversion', canBeginConversion, mediaUploadToken, props.maybeSelectedVoiceConversionModel);
let noResultsSection = (
<div className="panel panel-inner text-center p-5 rounded-5 h-100">
<div className="d-flex flex-column opacity-75 h-100 justify-content-center">
<FontAwesomeIcon icon={faHeadphones} className="fs-3 mb-3" />
<h5 className="fw-semibold">
{t("common.SessionTtsInferenceResults.noResults.title")}
</h5>
<p>{t("common.SessionTtsInferenceResults.noResults.subtitle")}</p>
</div>
</div>
);
// let comingSoon = (
// <div className="overflow-hidden">
// <div className="panel panel-inner text-center p-5 rounded-5 h-100">
// <div className="d-flex flex-column opacity-75 h-100 justify-content-center">
// <FontAwesomeIcon icon={faTimer} className="fs-3 mb-3" />
// <p>This feature is coming soon!</p>
// </div>
// </div>
// </div>
// );
const enableConvertButton = canConvert
&& mediaUploadToken !== undefined
&& props.maybeSelectedVoiceConversionModel !== undefined;
return (
<motion.div initial="hidden" animate="visible" variants={container}>
@@ -243,11 +212,15 @@ function VcModelListPage(props: Props) {
<div className="d-flex flex-column gap-3 upload-component">
<UploadComponent
setMediaUploadToken={setMediaUploadToken}
formIsCleared={formIsCleared}
setFormIsCleared={setFormIsCleared}
setCanConvert={setCanConvert}
changeConvertIdempotencyToken={changeConvertIdempotencyToken}
/>
</div>
</div>
<div>
{/*<div>
<label className="sub-title">
Or pick from your audio collection (5 files)
</label>
@@ -270,32 +243,31 @@ function VcModelListPage(props: Props) {
})}
/>
</div>
</div>
</div>*/}
<div>
<label className="sub-title">
Convert Audio
</label>
<div className="d-flex gap-3">
<button
className={speakButtonClass}
onClick={handleVoiceConversion}
type="submit"
disabled={!enableConvertButton}
>
<FontAwesomeIcon
icon={faRightLeft}
className="me-2"
/>
Convert
{loading && <LoadingIcon />}
</button>
</div>
</div>
<div className="d-flex gap-3">
<button
className={speakButtonClass}
onClick={handleVoiceConversion}
type="submit"
disabled={!canBeginConversion}
>
<FontAwesomeIcon
icon={faRightLeft}
className="me-2"
/>
Convert
{loading && <LoadingIcon />}
</button>
<button
className="btn btn-destructive w-100"
onClick={handleClearClick}
disabled={true}
>
<FontAwesomeIcon icon={faTrash} className="me-2" />
Clear
</button>
</div>
</div>
</div>
<div
@@ -308,33 +280,38 @@ function VcModelListPage(props: Props) {
<div>
<label className="sub-title">Record Audio</label>
<div className="d-flex flex-column gap-3 upload-component">
<RecordComponent />
<RecordComponent
setMediaUploadToken={setMediaUploadToken}
formIsCleared={formIsCleared}
setFormIsCleared={setFormIsCleared}
setCanConvert={setCanConvert}
changeConvertIdempotencyToken={changeConvertIdempotencyToken}
/>
</div>
</div>
<div className="d-flex gap-3">
<button
className={speakButtonClass}
onClick={handleLoading}
type="submit"
disabled={!canBeginConversion}
>
<FontAwesomeIcon
icon={faRightLeft}
className="me-2"
/>
Convert
{loading && <LoadingIcon />}
</button>
<button
className="btn btn-destructive w-100"
onClick={handleClearClick}
disabled={true}
>
<FontAwesomeIcon icon={faTrash} className="me-2" />
Clear
</button>
<div>
<label className="sub-title">
Convert Audio
</label>
<div className="d-flex gap-3">
<button
className={speakButtonClass}
onClick={handleVoiceConversion}
type="submit"
disabled={!enableConvertButton}
>
<FontAwesomeIcon
icon={faRightLeft}
className="me-2"
/>
Convert
{loading && <LoadingIcon />}
</button>
</div>
</div>
</div>
</div>
</div>
@@ -349,23 +326,10 @@ function VcModelListPage(props: Props) {
Session VC Results
</h4>
<div className="d-flex flex-column gap-3 session-tts-section">
{noResultsSection}
{/* <motion.div
className="panel panel-tts-results p-4 gap-3 d-flex flex-column"
variants={sessionItem}
>
<div>
<h5 className="mb-2">Title</h5>
<p>text</p>
</div>
(wavesurfer)
<div className="mt-2">
<Link to="/voice-conversion" className="fw-semibold">
<FontAwesomeIcon icon={faLink} className="me-2" />
Details
</Link>
</div>
</motion.div> */}
<SessionVoiceConversionResultsList
inferenceJobs={props.inferenceJobsByCategory.get(FrontendInferenceJobType.VoiceConversion)!}
sessionSubscriptionsWrapper={props.sessionSubscriptionsWrapper}
/>
</div>
</div>
</div>
@@ -1,67 +1,171 @@
import React, { useState } from "react";
import React, { useEffect, useMemo } from "react";
import { v4 as uuidv4 } from "uuid";
import { useAudioRecorder } from "react-audio-voice-recorder";
import { InputVcAudioPlayer } from "../../../../_common/InputVcAudioPlayer";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faMicrophone } from "@fortawesome/pro-solid-svg-icons";
import { faMicrophone, faUpload } from "@fortawesome/pro-solid-svg-icons";
import { UploadAudio, UploadAudioIsOk, UploadAudioRequest } from "@storyteller/components/src/api/upload/UploadAudio";
import { faTrash } from "@fortawesome/free-solid-svg-icons";
export default function RecordComponent() {
const [recorded, setRecorded] = useState(false);
const [loading, setLoading] = useState(false);
const [audioLink, setAudioLink] = useState<string>();
interface Props {
setMediaUploadToken: (token?: string) => void,
formIsCleared: boolean,
setFormIsCleared: (cleared: boolean) => void,
setCanConvert: (canConvert: boolean) => void,
changeConvertIdempotencyToken: () => void,
}
export default function RecordComponent(props: Props) {
const { startRecording, stopRecording, recordingBlob, isRecording } =
useAudioRecorder();
const addAudioElement = (blob: any) => {
stopRecording();
setLoading(true);
if (recordingBlob) {
const url = URL.createObjectURL(recordingBlob);
setAudioLink(url);
setRecorded(true);
setLoading(false);
useEffect(() => {
// NB: This is used to detect changes to `recordingBlob` and upload them
if (!recordingBlob) {
return;
}
//(async () => {
// let idempotencyToken = uuidv4();
// const request : UploadAudioRequest = {
// uuid_idempotency_token: idempotencyToken,
// file: recordingBlob,
// source: 'device',
// }
// let result = await UploadAudio(request);
// if (UploadAudioIsOk(result)) {
// //setIsUploadDisabled(true);
// //ggprops.setMediaUploadToken(result.upload_token);
// props.setMediaUploadToken(result.upload_token);
// }
//})();
}, [recordingBlob])
const handleStartRecording = async () => {
startRecording();
props.setCanConvert(false);
};
const handleStopRecording = async (blob: any) => {
stopRecording();
props.setFormIsCleared(false);
props.setCanConvert(false);
};
const handleClear = () => {
stopRecording();
props.setMediaUploadToken(undefined); // clear
props.setFormIsCleared(true);
props.setCanConvert(false);
props.changeConvertIdempotencyToken();
};
const handleUpload = async () => {
const request : UploadAudioRequest = {
uuid_idempotency_token: uuidv4(), // TODO: only send on change.
file: recordingBlob,
source: 'device',
}
let result = await UploadAudio(request);
if (UploadAudioIsOk(result)) {
props.setMediaUploadToken(result.upload_token);
props.setFormIsCleared(false);
props.setCanConvert(true);
props.changeConvertIdempotencyToken();
}
}
const enableMediaReview = !props.formIsCleared && recordingBlob !== undefined;
const enableUploadButton = !props.formIsCleared && recordingBlob !== undefined && !isRecording;
const speakButtonClass = enableUploadButton
? "btn btn-primary w-100"
: "btn btn-primary w-100 disabled";
return (
<div className="d-flex flex-column gap-3" id="record-audio">
{isRecording ? (
<button className="btn btn-secondary" onClick={addAudioElement}>
{loading ? (
<div className="d-flex align-items-center">
<div className="spinner-border spinner-border-sm" role="status">
<span className="visually-hidden">Saving...</span>
</div>
Saving
<button className="btn btn-secondary" onClick={handleStopRecording}>
<div className="d-flex align-items-center">
<div
className="spinner-grow spinner-grow-sm text-danger me-2"
role="status"
>
<span className="visually-hidden">Recording...</span>
</div>
) : (
<div className="d-flex align-items-center">
<div
className="spinner-grow spinner-grow-sm text-danger me-2"
role="status"
>
<span className="visually-hidden">Recording...</span>
</div>
Stop Recording
</div>
)}
Stop Recording
</div>
</button>
) : (
<button className="btn btn-secondary" onClick={startRecording}>
<button className="btn btn-secondary" onClick={handleStartRecording}>
<FontAwesomeIcon icon={faMicrophone} className="me-2" />
Start Recording
</button>
)}
{recorded ? (
<div className="panel panel-inner rounded p-3">
<InputVcAudioPlayer filename={audioLink as string} />
</div>
) : (
<></>
)}
{enableMediaReview ? (
<>
<RecordedAudioComponent recordingBlob={recordingBlob} />
<div className="d-flex gap-3">
<button
className={speakButtonClass}
onClick={handleUpload}
type="submit"
disabled={!enableUploadButton}
>
<FontAwesomeIcon
icon={faUpload}
className="me-2"
/>
Upload
</button>
<button
className="btn btn-destructive w-100"
onClick={handleClear}
>
<FontAwesomeIcon icon={faTrash} className="me-2" />
Clear
</button>
</div>
</>
) : (
<></>
)
}
</div>
);
}
interface RecorderProps {
recordingBlob: any;
}
function RecordedAudioComponent(props: RecorderProps) {
// Only generate the URL on change.
const audioLink = useMemo(() => {
if (!props.recordingBlob) {
return;
}
return URL.createObjectURL(props.recordingBlob);
}, [props.recordingBlob]);
if (!props.recordingBlob) {
return <></>;
}
return (
<div className="panel panel-inner rounded p-3">
<InputVcAudioPlayer filename={audioLink as string} />
</div>
);
}
@@ -1,4 +1,4 @@
import { faFileArrowUp, faFileAudio } from "@fortawesome/pro-solid-svg-icons";
import { faFileArrowUp, faFileAudio, faTrash } from "@fortawesome/pro-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import React, { useState } from "react";
import { FileUploader } from "react-drag-drop-files";
@@ -9,7 +9,13 @@ import { UploadAudio, UploadAudioIsOk, UploadAudioRequest } from "@storyteller/c
const FILE_TYPES = ["MP3", "WAV", "FLAC"];
interface Props {
setMediaUploadToken: (token: string) => void,
setMediaUploadToken: (token?: string) => void,
formIsCleared: boolean,
setFormIsCleared: (cleared: boolean) => void,
setCanConvert: (canConvert: boolean) => void,
changeConvertIdempotencyToken: () => void,
}
function UploadComponent(props: Props) {
@@ -21,10 +27,14 @@ function UploadComponent(props: Props) {
const [idempotencyToken, setIdempotencyToken] = useState(uuidv4());
const handleChange = (file: any) => {
console.log('handle change');
setFile(file);
setIdempotencyToken(uuidv4());
const audioUrl = URL.createObjectURL(file);
setAudioLink(audioUrl ?? "");
props.setFormIsCleared(false);
props.setCanConvert(false);
props.changeConvertIdempotencyToken();
};
const handleDragOver = (e: React.DragEvent<HTMLDivElement>): void => {
@@ -39,6 +49,16 @@ function UploadComponent(props: Props) {
e.currentTarget.classList.remove("upload-zone-drag");
};
const handleClear = () => {
setFile(null);
setAudioLink("");
setIdempotencyToken(uuidv4());
props.setMediaUploadToken(undefined); // clear
props.setFormIsCleared(true);
props.setCanConvert(false);
props.changeConvertIdempotencyToken();
};
const handleUploadFile = async () => {
if (file === undefined) {
return false;
@@ -47,6 +67,7 @@ function UploadComponent(props: Props) {
const request : UploadAudioRequest = {
uuid_idempotency_token: idempotencyToken,
file: file,
source: 'file',
}
let result = await UploadAudio(request);
@@ -54,7 +75,8 @@ function UploadComponent(props: Props) {
if (UploadAudioIsOk(result)) {
setIsUploadDisabled(true);
props.setMediaUploadToken(result.upload_token);
console.log('TOKEN', result.upload_token)
props.setFormIsCleared(false);
props.setCanConvert(true);
}
};
@@ -159,6 +181,14 @@ function UploadComponent(props: Props) {
/>
Upload Audio
</button>
<button
className="btn btn-destructive w-100"
onClick={handleClear}
>
<FontAwesomeIcon icon={faTrash} className="me-2" />
Clear
</button>
</div>
) : (
<></>