sound toggle for veo models (#877)

- veo models if selected now has a sound toggle button which defaults to
ON/true (still needs backend link)

frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx

@@ -6,7 +6,10 @@ import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover";
 import { Tooltip } from "@storyteller/ui-tooltip";
 import { Button, ToggleButton } from "@storyteller/ui-button";
 import { Modal } from "@storyteller/ui-modal";
-import { EnqueueImageToVideo, EnqueueImageToVideoRequest } from "@storyteller/tauri-api";
+import {
+  EnqueueImageToVideo,
+  EnqueueImageToVideoRequest,
+} from "@storyteller/tauri-api";
 import {
   faMessageXmark,
   faMessageCheck,
@@ -14,12 +17,17 @@ import {
   faSpinnerThird,
   faSquare,
   faPortrait,
+  faWaveformLines,
 } from "@fortawesome/pro-solid-svg-icons";
 import { faRectangle } from "@fortawesome/pro-regular-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { IconProp } from "@fortawesome/fontawesome-svg-core";
 import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
-import { SizeIconOption, SizeOption, VideoModel } from "@storyteller/model-list";
+import {
+  SizeIconOption,
+  SizeOption,
+  VideoModel,
+} from "@storyteller/model-list";
 import { usePromptVideoStore, RefImage } from "./promptStore";
 import { gtagEvent } from "@storyteller/google-analytics";
 import { ImagePromptRow } from "./ImagePromptRow";
@@ -27,7 +35,7 @@ import type { UploadImageFn } from "./ImagePromptRow";
 import { twMerge } from "tailwind-merge";
 import { toast } from "@storyteller/ui-toaster";
 
-const DEFAULT_RESOLUTIONS : SizeOption[] =  [
+const DEFAULT_RESOLUTIONS: SizeOption[] = [
   {
     tauriValue: "720p",
     textLabel: "720p",
@@ -83,6 +91,10 @@ export const PromptBoxVideo = ({
   const setPrompt = usePromptVideoStore((s) => s.setPrompt);
   const useSystemPrompt = usePromptVideoStore((s) => s.useSystemPrompt);
   const setUseSystemPrompt = usePromptVideoStore((s) => s.setUseSystemPrompt);
+  const generateWithSound = usePromptVideoStore((s) => s.generateWithSound);
+  const setGenerateWithSound = usePromptVideoStore(
+    (s) => s.setGenerateWithSound
+  );
   const resolution = usePromptVideoStore((s) => s.resolution);
   const setResolution = usePromptVideoStore((s) => s.setResolution);
   const [isEnqueueing, setIsEnqueueing] = useState(false);
@@ -109,7 +121,9 @@ export const PromptBoxVideo = ({
   if (!!selectedModel?.sizeOptions && selectedModel.sizeOptions.length > 0) {
     // When switching to a new model, the existing resolution might not be correct.
     // This is a gross and nasty hack to handle the case where the resolution is not found.
-    const resolutionExists = selectedModel.sizeOptions.some((option) => option.textLabel === resolution);
+    const resolutionExists = selectedModel.sizeOptions.some(
+      (option) => option.textLabel === resolution
+    );
     const useFirstOption = !resolutionExists;
 
     resolutionOptions = selectedModel.sizeOptions.map((option, index) => {
@@ -128,12 +142,15 @@ export const PromptBoxVideo = ({
       const icon = <FontAwesomeIcon icon={faIcon} className="h-4 w-4" />;
       return {
         label: option.textLabel,
-        selected: option.textLabel === resolution || (useFirstOption && index === 0),
+        selected:
+          option.textLabel === resolution || (useFirstOption && index === 0),
         icon: icon,
       };
     });
   } else {
-    const resolutionExists = DEFAULT_RESOLUTIONS.some((option) => option.textLabel === resolution);
+    const resolutionExists = DEFAULT_RESOLUTIONS.some(
+      (option) => option.textLabel === resolution
+    );
     const useFirstOption = !resolutionExists;
 
     resolutionOptions = DEFAULT_RESOLUTIONS.map((option, index) => {
@@ -152,13 +169,15 @@ export const PromptBoxVideo = ({
       const icon = <FontAwesomeIcon icon={faIcon} className="h-4 w-4" />;
       return {
         label: option.textLabel,
-        selected: option.textLabel === resolution || (useFirstOption && index === 0),
+        selected:
+          option.textLabel === resolution || (useFirstOption && index === 0),
         icon: icon,
       };
     });
   }
 
-  const [resolutionList, setResolutionList] = useState<PopoverItem[]>(resolutionOptions);
+  const [resolutionList, setResolutionList] =
+    useState<PopoverItem[]>(resolutionOptions);
 
   const textareaRef = useRef<HTMLTextAreaElement>(null);
 
@@ -252,7 +271,7 @@ export const PromptBoxVideo = ({
       setIsEnqueueing(false);
     }, 10000);
 
-    let request : EnqueueImageToVideoRequest = {
+    let request: EnqueueImageToVideoRequest = {
       model: selectedModel,
       image_media_token: imageMediaToken,
       prompt: prompt,
@@ -262,7 +281,16 @@ export const PromptBoxVideo = ({
     };
 
     if (selectedModel?.tauriId === "sora_2") {
-      request.sora_orientation = resolution === "720p" ? "landscape" : "portrait";
+      request.sora_orientation =
+        resolution === "720p" ? "landscape" : "portrait";
+    }
+
+    if (selectedModel?.tauriId === "veo_2") {
+      request.generate_with_sound = !!generateWithSound;
+    }
+
+    if (selectedModel?.tauriId === "veo_3") {
+      request.generate_with_sound = !!generateWithSound;
     }
 
     await EnqueueImageToVideo(request);
@@ -297,9 +325,8 @@ export const PromptBoxVideo = ({
 
   const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false);
 
-  const modelNeedsAnImageButNoneAreSelected = 
-    selectedModel?.requiresImage && 
-    referenceImages.length === 0;
+  const modelNeedsAnImageButNoneAreSelected =
+    selectedModel?.requiresImage && referenceImages.length === 0;
 
   // Hide/clear ending frame if model doesn't support it
   useEffect(() => {
@@ -421,6 +448,7 @@ export const PromptBoxVideo = ({
                   }
                 />
               </Tooltip>
+
               <Tooltip
                 content={
                   useSystemPrompt
@@ -438,6 +466,22 @@ export const PromptBoxVideo = ({
                   onClick={() => setUseSystemPrompt(!useSystemPrompt)}
                 />
               </Tooltip>
+
+              {selectedModel?.generateWithSound && (
+                <Tooltip
+                  content={generateWithSound ? "Sound: ON" : "Sound: OFF"}
+                  position="top"
+                  className="z-50"
+                  delay={200}
+                >
+                  <ToggleButton
+                    isActive={generateWithSound}
+                    icon={faWaveformLines}
+                    activeIcon={faWaveformLines}
+                    onClick={() => setGenerateWithSound(!generateWithSound)}
+                  />
+                </Tooltip>
+              )}
             </div>
             <div className="flex items-center gap-2">
               <Button

frontend/libs/components/promptbox/src/lib/promptStore.ts

@@ -93,11 +93,13 @@ interface PromptVideoStore {
   useSystemPrompt: boolean;
   referenceImages: RefImage[];
   endFrameImage?: RefImage;
+  generateWithSound: boolean;
   setPrompt: (prompt: string) => void;
   setResolution: (resolution: Resolution | string) => void;
   setUseSystemPrompt: (value: boolean) => void;
   setReferenceImages: (images: RefImage[]) => void;
   setEndFrameImage: (image?: RefImage) => void;
+  setGenerateWithSound: (value: boolean) => void;
 }
 
 export const usePromptVideoStore = create<PromptVideoStore>()((set) => ({
@@ -106,11 +108,13 @@ export const usePromptVideoStore = create<PromptVideoStore>()((set) => ({
   useSystemPrompt: true,
   referenceImages: [],
   endFrameImage: undefined,
+  generateWithSound: true,
   setPrompt: (prompt) => set({ prompt }),
   setResolution: (resolution) => set({ resolution }),
   setUseSystemPrompt: (useSystemPrompt) => set({ useSystemPrompt }),
   setReferenceImages: (referenceImages) => set({ referenceImages }),
   setEndFrameImage: (endFrameImage) => set({ endFrameImage }),
+  setGenerateWithSound: (generateWithSound) => set({ generateWithSound }),
 }));
 
 // ----- Edit Prompt Box Store -----

frontend/libs/model-list/src/lib/classes/VideoModel.ts

@@ -22,6 +22,9 @@ export class VideoModel extends Model {
   // The size options for the model
   readonly sizeOptions: SizeOption[];
 
+  // Whether this model supports toggling generation with sound
+  readonly generateWithSound?: boolean;
+
   constructor(args: {
     id: string;
     tauriId: string;
@@ -37,11 +40,13 @@ export class VideoModel extends Model {
     tags?: ModelTag[];
     sizeOptions?: SizeOption[];
     progressBarTime?: number;
+    generateWithSound?: boolean;
   }) {
     super(args);
     this.startFrame = args.startFrame;
     this.endFrame = args.endFrame;
     this.requiresImage = args.requiresImage;
     this.sizeOptions = args.sizeOptions ?? [];
+    this.generateWithSound = args.generateWithSound;
   }
 }

frontend/libs/model-list/src/lib/lists/VideoModels.ts

@@ -97,6 +97,7 @@ export const VIDEO_MODELS: VideoModel[] = [
     startFrame: true,
     endFrame: false,
     requiresImage: true,
+    generateWithSound: true,
     progressBarTime: 100000,
   }),
   new VideoModel({
@@ -111,6 +112,7 @@ export const VIDEO_MODELS: VideoModel[] = [
     startFrame: true,
     endFrame: false,
     requiresImage: true,
+    generateWithSound: true,
     progressBarTime: 100000,
   }),
   new VideoModel({
@@ -125,6 +127,7 @@ export const VIDEO_MODELS: VideoModel[] = [
     startFrame: true,
     endFrame: false,
     requiresImage: true,
+    generateWithSound: true,
     progressBarTime: 100000,
   }),
 ];

frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageToVideo.ts

@@ -34,6 +34,9 @@ export interface EnqueueImageToVideoRequest {
 
   // Optional. Orientation of the video for Sora 2.
   sora_orientation?: "portrait" | "landscape";
+
+  // Optional. Whether to generate audio alongside the video (used by some models like Veo2)
+  generate_with_sound?: boolean;
 }
 
 interface RawEnqueueImageToVideoRequest {
@@ -44,6 +47,7 @@ interface RawEnqueueImageToVideoRequest {
   frontend_caller?: string;
   frontend_subscriber_id?: string;
   sora_orientation?: "portrait" | "landscape";
+  generate_with_sound?: boolean;
 }
 
 export interface EnqueueImageToVideoError extends CommandResult {
@@ -51,7 +55,7 @@ export interface EnqueueImageToVideoError extends CommandResult {
   error_message?: string;
 }
 
-export interface EnqueueImageToVideoPayload {}
+export type EnqueueImageToVideoPayload = Record<string, never>;
 
 export interface EnqueueImageToVideoSuccess extends CommandResult {
   payload: EnqueueImageToVideoPayload;
@@ -64,32 +68,36 @@ export type EnqueueImageToVideoResult =
 export const EnqueueImageToVideo = async (
   request: EnqueueImageToVideoRequest
 ): Promise<EnqueueImageToVideoResult> => {
-  let mutableRequest: RawEnqueueImageToVideoRequest = {
+  const mutableRequest: RawEnqueueImageToVideoRequest = {
     model: request.model?.tauriId,
     image_media_token: request.image_media_token,
   };
 
-  if (!!request.prompt) {
+  if (request.prompt) {
     mutableRequest.prompt = request.prompt;
   }
 
-  if (!!request.end_frame_image_media_token) {
+  if (request.end_frame_image_media_token) {
     mutableRequest.end_frame_image_media_token =
       request.end_frame_image_media_token;
   }
 
-  if (!!request.frontend_caller) {
+  if (request.frontend_caller) {
     mutableRequest.frontend_caller = request.frontend_caller;
   }
 
-  if (!!request.frontend_subscriber_id) {
+  if (request.frontend_subscriber_id) {
     mutableRequest.frontend_subscriber_id = request.frontend_subscriber_id;
   }
 
-  if (!!request.sora_orientation) {
+  if (request.sora_orientation) {
     mutableRequest.sora_orientation = request.sora_orientation;
   }
 
+  if (typeof request.generate_with_sound === "boolean") {
+    mutableRequest.generate_with_sound = request.generate_with_sound;
+  }
+
   const result = await invoke("enqueue_image_to_video_command", {
     request: mutableRequest,
   });