diff --git a/crates/desktop/artcraft/src/commands/app_preferences/get_app_preferences_command.rs b/crates/desktop/artcraft/src/commands/app_preferences/get_app_preferences_command.rs index 4d2fabb490..7c5b58e611 100644 --- a/crates/desktop/artcraft/src/commands/app_preferences/get_app_preferences_command.rs +++ b/crates/desktop/artcraft/src/commands/app_preferences/get_app_preferences_command.rs @@ -17,7 +17,15 @@ pub struct AppPreferencesPayload { /// Play sounds on events. pub play_sounds: bool, - + + /// Key pointing to file; defined in the frontend code. + /// Defined for enqueue since image enqueue can be async + pub enqueue_success_sound: Option, + + /// Key pointing to file; defined in the frontend code. + /// Defined for enqueue since image enqueue can be async + pub enqueue_failure_sound: Option, + /// Key pointing to file; defined in the frontend code. pub generation_success_sound: Option, @@ -25,6 +33,7 @@ pub struct AppPreferencesPayload { pub generation_failure_sound: Option, /// Key pointing to file; defined in the frontend code. + #[deprecated] pub generation_enqueue_sound: Option, } @@ -52,6 +61,8 @@ async fn get_prefs(app_prefs: &AppPreferencesManager) -> AnyhowResult { /// Both "success" and "error" types have a `status` field for the frontend. pub status: CommandSuccessStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub success_message: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub payload: Option, } @@ -36,7 +39,10 @@ pub struct CommandErrorResponseWrapper { pub status: CommandErrorStatus, #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, + pub error_message: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub error_details: Option, } @@ -44,6 +50,7 @@ impl CommandSuccessResponseWrapper<()> { pub fn empty() -> Self { CommandSuccessResponseWrapper { status: CommandSuccessStatus::Success, + success_message: None, payload: None, } } @@ -61,6 +68,7 @@ impl From<()> for CommandSuccessResponseWrapper<()> { fn from(_: ()) -> Self { CommandSuccessResponseWrapper { status: CommandSuccessStatus::Success, + success_message: None, payload: None, } } @@ -72,16 +80,28 @@ impl From for CommandSuccessResponseWrapper where T: Serialize + Seria fn from(val: T) -> Self { CommandSuccessResponseWrapper { status: CommandSuccessStatus::Success, + success_message: None, payload: Some(val), } } } +impl From<&str> for CommandSuccessResponseWrapper { + fn from(msg: &str) -> Self { + CommandSuccessResponseWrapper { + status: CommandSuccessStatus::Success, + success_message: Some(msg.to_string()), + payload: None, + } + } +} + impl From<&str> for CommandErrorResponseWrapper { fn from(value: &str) -> Self { CommandErrorResponseWrapper { status: CommandErrorStatus::BadRequest, // NB: Default to 500 error. - error: Some(value.to_string()), + error_message: Some(value.to_string()), + error_details: None, } } } diff --git a/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs b/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs index f5163345e7..26b84e2667 100644 --- a/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs +++ b/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs @@ -1,3 +1,4 @@ +use crate::commands::command_response_wrapper::CommandResult; use crate::events::sendable_event_trait::SendableEvent; use crate::events::sora::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; use crate::events::sora::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; @@ -56,7 +57,7 @@ pub async fn sora_image_remix_command( app_data_root: State<'_, AppDataRoot>, sora_creds_manager: State<'_, SoraCredentialManager>, sora_task_queue: State<'_, SoraTaskQueue>, -) -> Result { +) -> CommandResult<(), String> { info!("image_generation_command called; scene media token: {:?}, additional images: {:?}", request.snapshot_media_token, request.maybe_additional_images); @@ -75,7 +76,7 @@ pub async fn sora_image_remix_command( error!("Failed to emit event: {:?}", err); } - return Err("there was an error".to_string()); + return Err("there was an error".into()); } let event = SoraImageEnqueueSuccessEvent {}; @@ -85,7 +86,7 @@ pub async fn sora_image_remix_command( error!("Failed to emit event: {:?}", err); } - Ok("success".to_string()) + Ok(().into()) } pub async fn generate_image( diff --git a/crates/desktop/artcraft/src/state/app_preferences/app_preferences.rs b/crates/desktop/artcraft/src/state/app_preferences/app_preferences.rs index 3dc1d00968..30a2b72910 100644 --- a/crates/desktop/artcraft/src/state/app_preferences/app_preferences.rs +++ b/crates/desktop/artcraft/src/state/app_preferences/app_preferences.rs @@ -9,6 +9,14 @@ pub struct AppPreferences { /// Play sounds on events. pub play_sounds: bool, + + /// Key pointing to file; defined in the frontend code. + /// Defined for enqueue since image enqueue can be async + pub enqueue_success_sound: Option, + + /// Key pointing to file; defined in the frontend code. + /// Defined for enqueue since image enqueue can be async + pub enqueue_failure_sound: Option, /// Key pointing to file; defined in the frontend code. pub generation_success_sound: Option, @@ -17,6 +25,7 @@ pub struct AppPreferences { pub generation_failure_sound: Option, /// Key pointing to file; defined in the frontend code. + #[deprecated] pub generation_enqueue_sound: Option, } @@ -26,7 +35,9 @@ impl Default for AppPreferences { preferred_download_directory: PreferredDownloadDirectory::System(SystemDownloadDirectory::Downloads), play_sounds: true, // NB: These are defined in the frontend. - generation_success_sound: Some("flower".to_string()), + enqueue_success_sound: Some("done".to_string()), + enqueue_failure_sound: Some("spike_throw".to_string()), + generation_success_sound: Some("special_flower".to_string()), generation_failure_sound: Some("crumble".to_string()), generation_enqueue_sound: Some("done".to_string()), } diff --git a/frontend/apps/editor3d/app/public/resources/sound/goldensun_101.wav b/frontend/apps/editor3d/app/public/resources/sound/goldensun_101.wav new file mode 100644 index 0000000000..ecdaa953cb Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/goldensun_101.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/goldensun_102.wav b/frontend/apps/editor3d/app/public/resources/sound/goldensun_102.wav new file mode 100644 index 0000000000..d508557bd7 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/goldensun_102.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/goldensun_111.wav b/frontend/apps/editor3d/app/public/resources/sound/goldensun_111.wav new file mode 100644 index 0000000000..c477f5c4d0 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/goldensun_111.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/goldensun_113.wav b/frontend/apps/editor3d/app/public/resources/sound/goldensun_113.wav new file mode 100644 index 0000000000..5ae208c9d1 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/goldensun_113.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/goldensun_135.wav b/frontend/apps/editor3d/app/public/resources/sound/goldensun_135.wav new file mode 100644 index 0000000000..30b04b46fa Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/goldensun_135.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/goldensun_173.wav b/frontend/apps/editor3d/app/public/resources/sound/goldensun_173.wav new file mode 100644 index 0000000000..70eeab321a Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/goldensun_173.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/goldensun_174.wav b/frontend/apps/editor3d/app/public/resources/sound/goldensun_174.wav new file mode 100644 index 0000000000..2845ef0dee Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/goldensun_174.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/goldensun_175.wav b/frontend/apps/editor3d/app/public/resources/sound/goldensun_175.wav new file mode 100644 index 0000000000..d0f5e053d7 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/goldensun_175.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/goldensun_214.wav b/frontend/apps/editor3d/app/public/resources/sound/goldensun_214.wav new file mode 100644 index 0000000000..fd58d5521e Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/goldensun_214.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/smrpg_enemy_spikethrow.wav b/frontend/apps/editor3d/app/public/resources/sound/smrpg_enemy_spikethrow.wav new file mode 100644 index 0000000000..a08dc150a9 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/smrpg_enemy_spikethrow.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/smrpg_specialflower.wav b/frontend/apps/editor3d/app/public/resources/sound/smrpg_specialflower.wav new file mode 100644 index 0000000000..003a20ee06 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/smrpg_specialflower.wav differ diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/InstallSounds.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/InstallSounds.ts index cdff89335d..9de089b70b 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/InstallSounds.ts +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/InstallSounds.ts @@ -3,13 +3,39 @@ import { SoundRegistry, SoundEffect } from "@storyteller/soundboard"; // TODO: This is maintained in two places. Here and MiscSettingsPane. export const InstallSounds = () => { const r = SoundRegistry.getInstance(); - if (r.hasSound("crumble")) { + if (r.hasSound("decline_normal")) { return; } - r.setSoundOnce("flower", new SoundEffect("resources/sound/smrpg_flower.wav", { defaultVolume: 0.1 })); - r.setSoundOnce("correct", new SoundEffect("resources/sound/smrpg_correct.wav", { defaultVolume: 0.3 })); + + // Good for simple immediate enqueue + r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.2 })); // DEFAULT + + // Good for simple immediate failure + r.setSoundOnce("error_chirp", new SoundEffect("resources/sound/goldensun_135.wav", { defaultVolume: 0.2 })); + r.setSoundOnce("spike_throw", new SoundEffect("resources/sound/smrpg_enemy_spikethrow.wav", { defaultVolume: 0.1 })); // DEFAULT + + // Good for simple async success + r.setSoundOnce("special_flower", new SoundEffect("resources/sound/smrpg_specialflower.wav", { defaultVolume: 0.2 })); // DEFAULT + + // Good for async errors + r.setSoundOnce("crumble", new SoundEffect("resources/sound/smrpg_drybones_crumble.wav", { defaultVolume: 0.1 })); // DEFAULT + r.setSoundOnce("special_alert", new SoundEffect("resources/sound/goldensun_214.wav", { defaultVolume: 0.2 })); + + // Good for menus r.setSoundOnce("next", new SoundEffect("resources/sound/oot_dialogue_next.wav", { defaultVolume: 0.2 })); - r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.2 })); - r.setSoundOnce("crumble", new SoundEffect("resources/sound/smrpg_drybones_crumble.wav", { defaultVolume: 0.1 })); + r.setSoundOnce("select", new SoundEffect("resources/sound/goldensun_111.wav", { defaultVolume: 0.4 })); + + // Good for special reward / celebration + r.setSoundOnce("correct", new SoundEffect("resources/sound/smrpg_correct.wav", { defaultVolume: 0.1 })); + r.setSoundOnce("flower", new SoundEffect("resources/sound/smrpg_flower.wav", { defaultVolume: 0.1 })); + + // Misc / ungrouped + r.setSoundOnce("accept_chirp", new SoundEffect("resources/sound/goldensun_101.wav", { defaultVolume: 0.2 })); + r.setSoundOnce("accept_normal_level_1", new SoundEffect("resources/sound/goldensun_173.wav", { defaultVolume: 0.2 })); + r.setSoundOnce("accept_normal_level_2", new SoundEffect("resources/sound/goldensun_174.wav", { defaultVolume: 0.2 })); + r.setSoundOnce("accept_normal_level_3", new SoundEffect("resources/sound/goldensun_175.wav", { defaultVolume: 0.2 })); + r.setSoundOnce("decline_chirp", new SoundEffect("resources/sound/goldensun_102.wav", { defaultVolume: 0.2 })); + r.setSoundOnce("decline_normal", new SoundEffect("resources/sound/goldensun_113.wav", { defaultVolume: 0.2 })); + (window as any).sounds = r; } diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationFailure.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationFailure.ts index 1a74bdf654..8a3163cf13 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationFailure.ts +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationFailure.ts @@ -9,13 +9,9 @@ type SoraImageGenerationFailed = { export const InstallImageGenerationFailure = () => { listen('sora-image-generation-failed', async (event) => { - console.log("sora-image-generation-failed event", event); const prefs = await GetAppPreferences(); - console.log("prefs", prefs); const soundName = prefs.preferences?.generation_failure_sound; - console.log("soundName", soundName); if (soundName !== undefined) { - console.log("playing sound", soundName); const registry = SoundRegistry.getInstance(); registry.playSound(soundName); } diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationSuccess.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationSuccess.ts index 126622597d..da4e91cb58 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationSuccess.ts +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationSuccess.ts @@ -9,13 +9,9 @@ type ImageGenerationSuccess = { export const InstallImageGenerationSuccess = () => { listen('sora-image-generation-complete', async (event) => { - console.log("sora-image-generation-complete event", event); const prefs = await GetAppPreferences(); - console.log("prefs", prefs); const soundName = prefs.preferences?.generation_success_sound; - console.log("soundName", soundName); if (soundName !== undefined) { - console.log("playing sound", soundName); const registry = SoundRegistry.getInstance(); registry.playSound(soundName); } diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/PromptBox.tsx b/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/PromptBox.tsx index 1db0c0f269..71f58e5f23 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/PromptBox.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/PromptBox.tsx @@ -46,6 +46,8 @@ import Queue from "~/pages/PageEnigma/Queue/Queue"; import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; import { PromptsApi } from "@storyteller/api"; import { SoundRegistry } from "@storyteller/soundboard"; +import { GetAppPreferences, SoraImageRemix } from "@storyteller/tauri-api"; +import { CommandSuccessStatus } from "@storyteller/tauri-api"; import { cameras, @@ -522,19 +524,54 @@ export const PromptBox = () => { //sceneMediaToken: "", }); - const generateResponse = await invoke("sora_image_remix_command", { - request: { - snapshot_media_token: snapshotResult.data!, - disable_system_prompt: !useSystemPrompt, - prompt: prompt, - maybe_additional_images: referenceImages.map( - (image) => image.mediaToken, - ), - maybe_number_of_samples: 1, - }, + console.log("snapshotResult", snapshotResult) + + const generateResponse = await SoraImageRemix({ + snapshot_media_token: snapshotResult.data!, + disable_system_prompt: !useSystemPrompt, + prompt: prompt, + maybe_additional_images: referenceImages.map( + (image) => image.mediaToken, + ), + maybe_number_of_samples: 1, }); - console.log("response", generateResponse); + console.log("generateResponse", generateResponse) + + if (generateResponse.status === CommandSuccessStatus.Success) { + const prefs = await GetAppPreferences(); + const soundName = prefs.preferences?.enqueue_success_sound; + if (soundName !== undefined) { + const registry = SoundRegistry.getInstance(); + registry.playSound(soundName); + } + toast.success("Image added to queue"); + setisEnqueueing(false); + return; + + } else { + const prefs = await GetAppPreferences(); + const soundName = prefs.preferences?.enqueue_failure_sound; + if (soundName !== undefined) { + const registry = SoundRegistry.getInstance(); + registry.playSound(soundName); + } + toast.error("Failed to enqueue image"); + setisEnqueueing(false); + return; + } + + // const generateResponse = await invoke("sora_image_remix_command", { + // request: { + // snapshot_media_token: snapshotResult.data!, + // disable_system_prompt: !useSystemPrompt, + // prompt: prompt, + // maybe_additional_images: referenceImages.map( + // (image) => image.mediaToken, + // ), + // maybe_number_of_samples: 1, + // }, + // }); //if (generateResponse.errorMessage) { // handleError(response.errorMessage); diff --git a/frontend/libs/components/login-modal/tsconfig.json b/frontend/libs/components/login-modal/tsconfig.json index 092d507d96..1d1824cf3d 100644 --- a/frontend/libs/components/login-modal/tsconfig.json +++ b/frontend/libs/components/login-modal/tsconfig.json @@ -2,6 +2,9 @@ "files": [], "include": [], "references": [ + { + "path": "../../api" + }, { "path": "../../tauri-api" }, @@ -11,6 +14,9 @@ { "path": "../button" }, + { + "path": "../input" + }, { "path": "./tsconfig.lib.json" }, diff --git a/frontend/libs/components/login-modal/tsconfig.lib.json b/frontend/libs/components/login-modal/tsconfig.lib.json index f98ce3da89..e0ec84e7f8 100644 --- a/frontend/libs/components/login-modal/tsconfig.lib.json +++ b/frontend/libs/components/login-modal/tsconfig.lib.json @@ -43,6 +43,9 @@ "src/**/*.tsx" ], "references": [ + { + "path": "../../api/tsconfig.lib.json" + }, { "path": "../../tauri-api/tsconfig.lib.json" }, @@ -51,6 +54,9 @@ }, { "path": "../button/tsconfig.lib.json" + }, + { + "path": "../input/tsconfig.lib.json" } ] } diff --git a/frontend/libs/components/settings-modal/src/lib/panes/AudioSettingsPane.tsx b/frontend/libs/components/settings-modal/src/lib/panes/AudioSettingsPane.tsx index 100dbe999f..b6037e6b37 100644 --- a/frontend/libs/components/settings-modal/src/lib/panes/AudioSettingsPane.tsx +++ b/frontend/libs/components/settings-modal/src/lib/panes/AudioSettingsPane.tsx @@ -13,12 +13,19 @@ import { Label } from "@storyteller/ui-label"; // TODO: This is maintained in two places. Here and InstallSounds. const SOUND_OPTIONS = [ - { value: "none", label: "None (Silent)" }, - { value: "flower", label: "Flower" }, + { value: "none", label: "None (Silent)" }, // NB: MUST BE FIRST + + // Other options { value: "correct", label: "Correct" }, - { value: "next", label: "Next" }, - { value: "done", label: "Done" }, { value: "crumble", label: "Crumble" }, + { value: "done", label: "Done" }, + { value: "error_chirp", label: "Error Chirp" }, + { value: "flower", label: "Flower" }, + { value: "next", label: "Next" }, + { value: "select", label: "Select" }, + { value: "special_alert", label: "Special Alert" }, + { value: "special_flower", label: "Special Flower" }, + { value: "spike_throw", label: "Spike Throw" }, ]; interface AudioSettingsPaneProps {} @@ -38,9 +45,10 @@ export const AudioSettingsPane = (args: AudioSettingsPaneProps) => { const playSounds = preferences?.play_sounds || false; - const successSound = orNone(preferences?.generation_success_sound); - const failureSound = orNone(preferences?.generation_failure_sound); - const enqueueSound = orNone(preferences?.generation_enqueue_sound); + const enqueueSuccessSound = orNone(preferences?.enqueue_success_sound); + const enqueueFailureSound = orNone(preferences?.enqueue_failure_sound); + const generationSuccessSound = orNone(preferences?.generation_success_sound); + const generationFailureSound = orNone(preferences?.generation_failure_sound); const reloadPreferences = async () => { const prefs = await GetAppPreferences(); @@ -56,6 +64,26 @@ export const AudioSettingsPane = (args: AudioSettingsPaneProps) => { await reloadPreferences(); }; + const setEnqueueSuccessSound = async (val: string) => { + let sendVal = val === "none" ? undefined : val; + await UpdateAppPreferences({ + preference: PreferenceName.EnqueueSuccessSound, + value: sendVal, + }); + SoundRegistry.getInstance().playSound(val); + await reloadPreferences(); + }; + + const setEnqueueFailureSound = async (val: string) => { + let sendVal = val === "none" ? undefined : val; + await UpdateAppPreferences({ + preference: PreferenceName.EnqueueFailureSound, + value: sendVal, + }); + SoundRegistry.getInstance().playSound(val); + await reloadPreferences(); + }; + const setSuccessSound = async (val: string) => { let sendVal = val === "none" ? undefined : val; await UpdateAppPreferences({ @@ -76,16 +104,6 @@ export const AudioSettingsPane = (args: AudioSettingsPaneProps) => { await reloadPreferences(); }; - const setEnqueueSound = async (val: string) => { - let sendVal = val === "none" ? undefined : val; - await UpdateAppPreferences({ - preference: PreferenceName.GenerationEnqueueSound, - value: sendVal, - }); - SoundRegistry.getInstance().playSound(val); - await reloadPreferences(); - }; - const playSound = (val?: string) => { if (val !== undefined && val !== "none") { SoundRegistry.getInstance().playSound(val); @@ -103,11 +121,49 @@ export const AudioSettingsPane = (args: AudioSettingsPaneProps) => {
- +
setEnqueueFailureSound(val as string)} + options={SOUND_OPTIONS} + className="grow" + /> +
+
+ +
+ +
+ setFailureSound(val as string)} options={SOUND_OPTIONS} className="grow" @@ -135,29 +191,11 @@ export const AudioSettingsPane = (args: AudioSettingsPaneProps) => { variant="primary" className="w-[40px] h-[40px]" icon={faPlay} - onClick={() => playSound(failureSound)} + onClick={() => playSound(generationFailureSound)} />
-
- -
-