From 4c1eb5fce1a72714a1a98dad9130fa91065dda0a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 3 Mar 2026 18:01:06 -0500 Subject: [PATCH] add prompt context items (video references) to Seedance 2.0 --- .../prompt_context_semantic_type.rs | 12 +++++++++++- .../seedance_2p0_multi_function_video_gen_handler.rs | 8 ++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/schema/public/enums/src/by_table/prompt_context_items/prompt_context_semantic_type.rs b/crates/schema/public/enums/src/by_table/prompt_context_items/prompt_context_semantic_type.rs index 769b8f2287..9d5dabafe9 100644 --- a/crates/schema/public/enums/src/by_table/prompt_context_items/prompt_context_semantic_type.rs +++ b/crates/schema/public/enums/src/by_table/prompt_context_items/prompt_context_semantic_type.rs @@ -19,6 +19,9 @@ pub enum PromptContextSemanticType { /// Image-to-Video ending frame VidEndFrame, + /// Reference image for video generation (e.g. Seedance "vidref" mode) + VidRef, + /// Source image, eg. for inpainting. Imgsrc, @@ -44,6 +47,7 @@ impl PromptContextSemanticType { match self { Self::VidStartFrame => "vid_start_frame", Self::VidEndFrame => "vid_end_frame", + Self::VidRef => "vidref", Self::Imgsrc => "imgsrc", Self::Imgmask => "imgmask", Self::Imgref => "imgref", @@ -57,6 +61,7 @@ impl PromptContextSemanticType { match value { "vid_start_frame" => Ok(Self::VidStartFrame), "vid_end_frame" => Ok(Self::VidEndFrame), + "vidref" => Ok(Self::VidRef), "imgsrc" => Ok(Self::Imgsrc), "imgmask" => Ok(Self::Imgmask), "imgref" => Ok(Self::Imgref), @@ -73,6 +78,7 @@ impl PromptContextSemanticType { BTreeSet::from([ Self::VidStartFrame, Self::VidEndFrame, + Self::VidRef, Self::Imgsrc, Self::Imgmask, Self::Imgref, @@ -95,6 +101,7 @@ mod tests { fn test_serialization() { assert_serialization(PromptContextSemanticType::VidStartFrame, "vid_start_frame"); assert_serialization(PromptContextSemanticType::VidEndFrame, "vid_end_frame"); + assert_serialization(PromptContextSemanticType::VidRef, "vidref"); assert_serialization(PromptContextSemanticType::Imgsrc, "imgsrc"); assert_serialization(PromptContextSemanticType::Imgmask, "imgmask"); assert_serialization(PromptContextSemanticType::Imgref, "imgref"); @@ -111,6 +118,7 @@ mod tests { fn to_str() { assert_eq!(PromptContextSemanticType::VidStartFrame.to_str(), "vid_start_frame"); assert_eq!(PromptContextSemanticType::VidEndFrame.to_str(), "vid_end_frame"); + assert_eq!(PromptContextSemanticType::VidRef.to_str(), "vidref"); assert_eq!(PromptContextSemanticType::Imgsrc.to_str(), "imgsrc"); assert_eq!(PromptContextSemanticType::Imgmask.to_str(), "imgmask"); assert_eq!(PromptContextSemanticType::Imgref.to_str(), "imgref"); @@ -123,6 +131,7 @@ mod tests { fn from_str() { assert_eq!(PromptContextSemanticType::from_str("vid_start_frame").unwrap(), PromptContextSemanticType::VidStartFrame); assert_eq!(PromptContextSemanticType::from_str("vid_end_frame").unwrap(), PromptContextSemanticType::VidEndFrame); + assert_eq!(PromptContextSemanticType::from_str("vidref").unwrap(), PromptContextSemanticType::VidRef); assert_eq!(PromptContextSemanticType::from_str("imgsrc").unwrap(), PromptContextSemanticType::Imgsrc); assert_eq!(PromptContextSemanticType::from_str("imgmask").unwrap(), PromptContextSemanticType::Imgmask); assert_eq!(PromptContextSemanticType::from_str("imgref").unwrap(), PromptContextSemanticType::Imgref); @@ -139,9 +148,10 @@ mod tests { #[test] fn all_variants() { let mut variants = PromptContextSemanticType::all_variants(); - assert_eq!(variants.len(), 8); + assert_eq!(variants.len(), 9); assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::VidStartFrame)); assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::VidEndFrame)); + assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::VidRef)); assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgsrc)); assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgmask)); assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgref)); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs index 16d7c44baa..d8752c358a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs @@ -283,6 +283,14 @@ pub async fn seedance_2p0_multi_function_video_gen_handler( context_semantic_type: PromptContextSemanticType::VidEndFrame, }); } + if let Some(ref_tokens) = &request.reference_image_media_tokens { + for media_token in ref_tokens { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidRef, + }); + } + } if !context_items.is_empty() { if let Err(err) = insert_batch_prompt_context_items(InsertBatchArgs {