From df0a82cbdbee714521a0eda951f4ec5422af04ca Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 9 Jul 2026 21:00:32 -0400 Subject: [PATCH] triposplat support (#1764) --- .../src/api/router_splat_model.rs | 6 + .../generate_splat_request_builder.rs | 5 + .../generate_splat/generate_splat_response.rs | 31 +++ .../generate_splat/providers/artcraft/mod.rs | 1 + .../providers/artcraft/triposplat/build.rs | 14 ++ .../providers/artcraft/triposplat/cost.rs | 51 ++++ .../providers/artcraft/triposplat/mod.rs | 3 + .../providers/artcraft/triposplat/request.rs | 17 ++ .../generate_splat/providers/fal/mod.rs | 2 + .../providers/fal/request_common.rs | 44 ++++ .../providers/fal/triposplat/build.rs | 228 ++++++++++++++++++ .../providers/fal/triposplat/cost.rs | 59 +++++ .../providers/fal/triposplat/mod.rs | 3 + .../providers/fal/triposplat/request.rs | 62 +++++ .../generate/generate_splat/providers/mod.rs | 1 + .../splat_generation_request.rs | 18 ++ .../fal_client/src/requests/api/mod.rs | 1 + .../src/requests/api/splat/image/mod.rs | 1 + .../image/triposplat_image_to_splat/api.rs | 160 ++++++++++++ .../image/triposplat_image_to_splat/cost.rs | 47 ++++ .../image/triposplat_image_to_splat/mod.rs | 3 + .../triposplat_image_to_splat/raw_request.rs | 37 +++ .../fal_client/src/requests/api/splat/mod.rs | 1 + .../mimetypes/src/custom/custom_infer.rs | 22 ++ .../src/mimetype_info/file_extension.rs | 3 + .../list_user_bookmarks_result.rs | 1 + .../inference_model_type.rs | 9 + .../by_table/media_files/media_file_type.rs | 15 +- .../common/generation/common_model_type.rs | 12 +- .../common/generation/common_splat_model.rs | 9 + .../src/configs/omni_gen/splat_models.rs | 20 ++ .../omni_gen/generate/splat/helpers/mod.rs | 1 + .../helpers/resolve_media_tokens_to_urls.rs | 38 +++ .../splat/insert_db_job/insert_fal_job.rs | 56 +++++ .../generate/splat/insert_db_job/mod.rs | 1 + .../splat/omni_gen_splat_generate_handler.rs | 30 +++ .../splat/pipeline_v2/run_pipeline_v2.rs | 14 +- .../splat/validate_splat_request.rs | 26 +- .../process_model_mesh_payload.rs | 22 +- .../process_success/resolve_file_metadata.rs | 1 + 40 files changed, 1065 insertions(+), 10 deletions(-) create mode 100644 crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/build.rs create mode 100644 crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/cost.rs create mode 100644 crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/mod.rs create mode 100644 crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/request.rs create mode 100644 crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/mod.rs create mode 100644 crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/request_common.rs create mode 100644 crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/build.rs create mode 100644 crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/cost.rs create mode 100644 crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/mod.rs create mode 100644 crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/request.rs create mode 100644 crates/api_clients/fal_client/src/requests/api/splat/image/mod.rs create mode 100644 crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/api.rs create mode 100644 crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/cost.rs create mode 100644 crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/mod.rs create mode 100644 crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/raw_request.rs create mode 100644 crates/api_clients/fal_client/src/requests/api/splat/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/helpers/resolve_media_tokens_to_urls.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/insert_db_job/insert_fal_job.rs diff --git a/crates/api_clients/artcraft/artcraft_router/src/api/router_splat_model.rs b/crates/api_clients/artcraft/artcraft_router/src/api/router_splat_model.rs index d357da1027..606614018d 100644 --- a/crates/api_clients/artcraft/artcraft_router/src/api/router_splat_model.rs +++ b/crates/api_clients/artcraft/artcraft_router/src/api/router_splat_model.rs @@ -24,6 +24,10 @@ pub enum RouterSplatModel { #[serde(rename = "marble_1p1_plus")] Marble1p1Plus, + + /// TripoSplat: single-image to 3D Gaussian splat (via Fal). + #[serde(rename = "triposplat")] + TripoSplat, } #[cfg(test)] @@ -40,6 +44,7 @@ mod tests { assert_serde_round_trip(RouterSplatModel::Marble1p0Draft, "marble_1p0_draft"); assert_serde_round_trip(RouterSplatModel::Marble1p1, "marble_1p1"); assert_serde_round_trip(RouterSplatModel::Marble1p1Plus, "marble_1p1_plus"); + assert_serde_round_trip(RouterSplatModel::TripoSplat, "triposplat"); } #[test] @@ -53,6 +58,7 @@ mod tests { (RouterSplatModel::Marble1p0Draft, CommonSplatModel::Marble1p0Draft), (RouterSplatModel::Marble1p1, CommonSplatModel::Marble1p1), (RouterSplatModel::Marble1p1Plus, CommonSplatModel::Marble1p1Plus), + (RouterSplatModel::TripoSplat, CommonSplatModel::TripoSplat), ]; for (router_model, expected_common) in cases { let json = serde_json::to_string(&router_model).unwrap(); diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/generate_splat_request_builder.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/generate_splat_request_builder.rs index 32778cfadd..ea090e8f80 100644 --- a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/generate_splat_request_builder.rs +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/generate_splat_request_builder.rs @@ -8,6 +8,8 @@ use crate::generate::generate_splat::providers::artcraft::marble_1p0::build::bui use crate::generate::generate_splat::providers::artcraft::marble_1p0_draft::build::build_artcraft_marble_1p0_draft; use crate::generate::generate_splat::providers::artcraft::marble_1p1::build::build_artcraft_marble_1p1; use crate::generate::generate_splat::providers::artcraft::marble_1p1_plus::build::build_artcraft_marble_1p1_plus; +use crate::generate::generate_splat::providers::artcraft::triposplat::build::build_artcraft_triposplat; +use crate::generate::generate_splat::providers::fal::triposplat::build::build_fal_triposplat; use crate::generate::generate_splat::providers::worldlabs::marble_1p0::build::build_worldlabs_marble_1p0; use crate::generate::generate_splat::providers::worldlabs::marble_1p0_draft::build::build_worldlabs_marble_1p0_draft; use crate::generate::generate_splat::providers::worldlabs::marble_1p1::build::build_worldlabs_marble_1p1; @@ -79,6 +81,9 @@ impl GenerateSplatRequestBuilder { (RouterProvider::Artcraft, RouterSplatModel::Marble1p0Draft | RouterSplatModel::Marble0p1Mini) => build_artcraft_marble_1p0_draft(self), (RouterProvider::Artcraft, RouterSplatModel::Marble1p1) => build_artcraft_marble_1p1(self), (RouterProvider::Artcraft, RouterSplatModel::Marble1p1Plus) => build_artcraft_marble_1p1_plus(self), + (RouterProvider::Artcraft, RouterSplatModel::TripoSplat) => build_artcraft_triposplat(self), + // Fal + (RouterProvider::Fal, RouterSplatModel::TripoSplat) => build_fal_triposplat(self), // World Labs (RouterProvider::WorldLabs, RouterSplatModel::Marble1p0 | RouterSplatModel::Marble0p1Plus) => build_worldlabs_marble_1p0(self), (RouterProvider::WorldLabs, RouterSplatModel::Marble1p0Draft | RouterSplatModel::Marble0p1Mini) => build_worldlabs_marble_1p0_draft(self), diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/generate_splat_response.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/generate_splat_response.rs index 16895a8b21..97812ed71e 100644 --- a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/generate_splat_response.rs +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/generate_splat_response.rs @@ -1,3 +1,6 @@ +use std::fmt::Debug; +use std::sync::Arc; + use tokens::tokens::generic_inference_jobs::InferenceJobToken; #[derive(Clone, Debug)] @@ -13,9 +16,30 @@ pub struct WorldLabsSplatResponsePayload { pub done: bool, } +/// Fal's queue/webhook submission receipt for a splat generation. +#[derive(Clone, Debug)] +pub struct FalSplatResponsePayload { + /// Fal's ID for the queued request. Present on queue submissions and on + /// most webhook submissions. + pub request_id: Option, + + /// Gateway request ID (webhook submissions). + pub gateway_request_id: Option, + + /// Queue-mode status polling URL. + pub maybe_status_url: Option, + + /// Queue-mode response URL. + pub maybe_response_url: Option, + + /// The outbound request we sent, for debug logging. + pub maybe_outbound_request: Option>, +} + #[derive(Clone, Debug)] pub enum GenerateSplatResponse { Artcraft(ArtcraftSplatResponsePayload), + Fal(FalSplatResponsePayload), WorldLabs(WorldLabsSplatResponsePayload), } @@ -27,6 +51,13 @@ impl GenerateSplatResponse { } } + pub fn get_fal_payload(&self) -> Option { + match self { + Self::Fal(p) => Some(p.clone()), + _ => None, + } + } + pub fn get_worldlabs_payload(&self) -> Option { match self { Self::WorldLabs(p) => Some(p.clone()), diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/mod.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/mod.rs index fe0b5ede6a..7db75e463b 100644 --- a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/mod.rs +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/mod.rs @@ -6,3 +6,4 @@ pub mod marble_1p0; pub mod marble_1p0_draft; pub mod marble_1p1; pub mod marble_1p1_plus; +pub mod triposplat; diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/build.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/build.rs new file mode 100644 index 0000000000..ed53fb234a --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/build.rs @@ -0,0 +1,14 @@ +use enums::common::generation::common_splat_model::CommonSplatModel as CommonSplatModelEnum; + +use crate::errors::artcraft_router_error::ArtcraftRouterError; +use crate::generate::generate_splat::generate_splat_request_builder::GenerateSplatRequestBuilder; +use crate::generate::generate_splat::providers::artcraft::build_common::build_artcraft_omni_splat_request; +use crate::generate::generate_splat::providers::artcraft::triposplat::request::ArtcraftTripoSplatRequestState; +use crate::generate::generate_splat::splat_generation_draft_or_request::SplatGenerationDraftOrRequest; +use crate::generate::generate_splat::splat_generation_request::SplatGenerationRequest; + +pub fn build_artcraft_triposplat(builder: GenerateSplatRequestBuilder) -> Result { + let request = build_artcraft_omni_splat_request(builder, CommonSplatModelEnum::TripoSplat)?; + let state = ArtcraftTripoSplatRequestState { request }; + Ok(SplatGenerationDraftOrRequest::Request(SplatGenerationRequest::ArtcraftTripoSplat(state))) +} diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/cost.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/cost.rs new file mode 100644 index 0000000000..ae3ea179a8 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/cost.rs @@ -0,0 +1,51 @@ +use crate::generate::generate_splat::providers::artcraft::cost_common::artcraft_splat_cost_estimate; +use crate::generate::generate_splat::providers::artcraft::triposplat::request::ArtcraftTripoSplatRequestState; +use crate::generate::generate_splat::splat_generation_cost_estimate::SplatGenerationCostEstimate; + +const FLAT_COST_IN_USD_CENTS: u64 = 7; + +/// TripoSplat via Artcraft is flat priced: it takes a single input image and +/// no option affects the price. +#[derive(Clone, Debug)] +pub struct ArtcraftTripoSplatCostState { + pub cost_in_usd_cents: u64, +} + +impl ArtcraftTripoSplatCostState { + pub fn from_request(_request: &ArtcraftTripoSplatRequestState) -> Self { + Self { cost_in_usd_cents: FLAT_COST_IN_USD_CENTS } + } + + pub fn estimate_cost(&self) -> SplatGenerationCostEstimate { + artcraft_splat_cost_estimate(self.cost_in_usd_cents) + } +} + +#[cfg(test)] +mod tests { + use tokens::tokens::media_files::MediaFileToken; + + use crate::api::image_list_ref::ImageListRef; + use crate::api::router_provider::RouterProvider; + use crate::api::router_splat_model::RouterSplatModel; + use crate::generate::generate_splat::generate_splat_request_builder::GenerateSplatRequestBuilder; + + #[test] + fn flat_seven_cents() { + let builder = GenerateSplatRequestBuilder { + model: RouterSplatModel::TripoSplat, + provider: RouterProvider::Artcraft, + reference_images: Some(ImageListRef::MediaFileTokens(vec![ + MediaFileToken::new("mf_test123".to_string()), + ])), + ..Default::default() + }; + let cost = builder.build2() + .expect("build should succeed") + .estimate_cost() + .expect("estimate should succeed") + .cost_in_usd_cents + .expect("cost should be present"); + assert_eq!(cost, 7); + } +} diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/mod.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/mod.rs new file mode 100644 index 0000000000..98150bd014 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/mod.rs @@ -0,0 +1,3 @@ +pub mod build; +pub mod cost; +pub mod request; diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/request.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/request.rs new file mode 100644 index 0000000000..0d2b8e3faf --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/artcraft/triposplat/request.rs @@ -0,0 +1,17 @@ +use artcraft_api_defs::omni_gen::cost_and_generate_requests::omni_gen_splat_cost_and_generate_request::OmniGenSplatCostAndGenerateRequest; + +use crate::client::router_artcraft_client::RouterArtcraftClient; +use crate::errors::artcraft_router_error::ArtcraftRouterError; +use crate::generate::generate_splat::generate_splat_response::GenerateSplatResponse; +use crate::generate::generate_splat::providers::artcraft::request_common::send_artcraft_omni_splat_request; + +#[derive(Clone, Debug)] +pub struct ArtcraftTripoSplatRequestState { + pub request: OmniGenSplatCostAndGenerateRequest, +} + +impl ArtcraftTripoSplatRequestState { + pub async fn send(&self, client: &RouterArtcraftClient) -> Result { + send_artcraft_omni_splat_request(&self.request, client).await + } +} diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/mod.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/mod.rs new file mode 100644 index 0000000000..14e263d977 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod request_common; +pub mod triposplat; diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/request_common.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/request_common.rs new file mode 100644 index 0000000000..0fbdfe10cb --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/request_common.rs @@ -0,0 +1,44 @@ +use std::fmt::Debug; +use std::sync::Arc; + +use fal_client::requests::traits::fal_endpoint_trait::FalEndpoint; + +use crate::client::router_fal_client::RouterFalClient; +use crate::errors::artcraft_router_error::ArtcraftRouterError; +use crate::generate::generate_splat::generate_splat_response::{ + FalSplatResponsePayload, GenerateSplatResponse, +}; + +/// Send a splat request to fal using the client's dispatch mode (webhook or +/// queue). Shared by all Fal splat request states. +pub(crate) async fn send_fal_splat_request( + request: &R, + client: &RouterFalClient, +) -> Result +where + R: FalEndpoint + Clone + Debug + Send + Sync + 'static, +{ + let outbound: Arc = Arc::new(request.clone()); + + let payload = if let Some(webhook_url) = &client.webhook_url { + let response = request.send_webhook_request(&client.api_key, webhook_url).await?; + FalSplatResponsePayload { + request_id: response.request_id, + gateway_request_id: response.gateway_request_id, + maybe_status_url: None, + maybe_response_url: None, + maybe_outbound_request: Some(outbound), + } + } else { + let response = request.send_queue_request(&client.api_key).await?; + FalSplatResponsePayload { + request_id: Some(response.request_id), + gateway_request_id: None, + maybe_status_url: Some(response.status_url), + maybe_response_url: Some(response.response_url), + maybe_outbound_request: Some(outbound), + } + }; + + Ok(GenerateSplatResponse::Fal(payload)) +} diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/build.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/build.rs new file mode 100644 index 0000000000..08f2e4a6a3 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/build.rs @@ -0,0 +1,228 @@ +use fal_client::requests::api::splat::image::triposplat_image_to_splat::api::{ + TripoSplatImageToSplatRequest, TripoSplatOutputFormat, +}; + +use crate::api::image_list_ref::ImageListRef; +use crate::client::request_mismatch_mitigation_strategy::RequestMismatchMitigationStrategy; +use crate::errors::artcraft_router_error::ArtcraftRouterError; +use crate::errors::client_error::ClientError; +use crate::generate::generate_mesh::providers::reject_unsupported::reject_unsupported_option; +use crate::generate::generate_splat::generate_splat_request_builder::GenerateSplatRequestBuilder; +use crate::generate::generate_splat::providers::fal::triposplat::request::FalTripoSplatRequestState; +use crate::generate::generate_splat::splat_generation_draft_or_request::SplatGenerationDraftOrRequest; +use crate::generate::generate_splat::splat_generation_request::SplatGenerationRequest; + +/// TripoSplat takes exactly one input image and nothing else. Prompts, +/// videos, panorama and recaption flags are unsupported: rejected under +/// `ErrorOut`, dropped otherwise. +pub fn build_fal_triposplat(builder: GenerateSplatRequestBuilder) -> Result { + let state = build_fal_triposplat_state(builder)?; + Ok(SplatGenerationDraftOrRequest::Request(SplatGenerationRequest::FalTripoSplat(state))) +} + +pub(crate) fn build_fal_triposplat_state( + mut builder: GenerateSplatRequestBuilder, +) -> Result { + let strategy = builder.request_mismatch_mitigation_strategy; + + reject_unsupported_option("prompt", builder.prompt.as_ref(), strategy)?; + reject_unsupported_option("reference_video", builder.reference_video.as_ref(), strategy)?; + if builder.is_panoramic == Some(true) { + reject_unsupported_option("is_panoramic", builder.is_panoramic.as_ref(), strategy)?; + } + reject_unsupported_option("disable_recaption", builder.disable_recaption.as_ref(), strategy)?; + + let image_url = plan_single_image_url(builder.reference_images.take(), strategy)? + .ok_or_else(|| { + ArtcraftRouterError::InvalidInput( + "TripoSplat requires an input image".to_string(), + ) + })?; + + let request = TripoSplatImageToSplatRequest { + image_url, + num_gaussians: None, + num_inference_steps: None, + guidance_scale: None, + // Always request PLY output — it's the format our pipeline stores. + output_format: Some(TripoSplatOutputFormat::Ply), + seed: None, + enable_safety_checker: None, + }; + Ok(FalTripoSplatRequestState { request }) +} + +/// Resolve the single input image from `reference_images`. More than one +/// image rejects under `ErrorOut`; the other strategies keep the first. +/// Fal only takes URLs; media file tokens are rejected. +fn plan_single_image_url( + reference_images: Option, + strategy: RequestMismatchMitigationStrategy, +) -> Result, ArtcraftRouterError> { + let urls = match reference_images { + None => return Ok(None), + Some(ImageListRef::Urls(urls)) => urls, + Some(ImageListRef::MediaFileTokens(tokens)) if tokens.is_empty() => return Ok(None), + Some(ImageListRef::MediaFileTokens(_)) => { + return Err(ArtcraftRouterError::Client(ClientError::FalOnlySupportsUrls)); + } + }; + + if urls.len() > 1 { + if let RequestMismatchMitigationStrategy::ErrorOut = strategy { + return Err(ArtcraftRouterError::Client(ClientError::ModelDoesNotSupportOption { + field: "reference_images", + value: format!("Expected at most 1 image, got {}", urls.len()), + })); + } + } + Ok(urls.into_iter().next()) +} + +#[cfg(test)] +mod tests { + use crate::api::router_provider::RouterProvider; + use crate::api::router_splat_model::RouterSplatModel; + use crate::api::video_ref::VideoRef; + + use super::*; + + const IMAGE_URL: &str = "https://example.com/object.png"; + + mod input_tests { + use super::*; + + #[test] + fn single_image_builds() { + let state = build_fal_triposplat_state(image_builder()).expect("build"); + assert_eq!(state.request.image_url, IMAGE_URL); + } + + #[test] + fn output_format_is_always_ply() { + let state = build_fal_triposplat_state(image_builder()).expect("build"); + assert_eq!(state.request.output_format, Some(TripoSplatOutputFormat::Ply)); + } + + #[test] + fn no_image_is_rejected() { + let result = build_fal_triposplat_state(base_builder()); + assert!(matches!(result, Err(ArtcraftRouterError::InvalidInput(_)))); + } + + #[test] + fn media_tokens_are_rejected() { + use tokens::tokens::media_files::MediaFileToken; + let builder = GenerateSplatRequestBuilder { + reference_images: Some(ImageListRef::MediaFileTokens(vec![ + MediaFileToken::new("mf_test123".to_string()), + ])), + ..base_builder() + }; + assert!(matches!( + build_fal_triposplat_state(builder), + Err(ArtcraftRouterError::Client(ClientError::FalOnlySupportsUrls)) + )); + } + + #[test] + fn multiple_images_error_out_under_error_out() { + let builder = GenerateSplatRequestBuilder { + reference_images: Some(ImageListRef::Urls(vec![ + IMAGE_URL.to_string(), + "https://example.com/second.png".to_string(), + ])), + request_mismatch_mitigation_strategy: RequestMismatchMitigationStrategy::ErrorOut, + ..base_builder() + }; + assert!(build_fal_triposplat_state(builder).is_err()); + } + + #[test] + fn multiple_images_keep_the_first_under_lenient_strategies() { + let builder = GenerateSplatRequestBuilder { + reference_images: Some(ImageListRef::Urls(vec![ + IMAGE_URL.to_string(), + "https://example.com/second.png".to_string(), + ])), + ..base_builder() + }; + let state = build_fal_triposplat_state(builder).expect("build"); + assert_eq!(state.request.image_url, IMAGE_URL); + } + } + + mod unsupported_option_tests { + use super::*; + + #[test] + fn unsupported_options_error_out_under_error_out() { + let cases = [ + GenerateSplatRequestBuilder { + prompt: Some("a cozy cabin".to_string()), + ..strict_image_builder() + }, + GenerateSplatRequestBuilder { + reference_video: Some(VideoRef::Url("https://example.com/v.mp4".to_string())), + ..strict_image_builder() + }, + GenerateSplatRequestBuilder { + is_panoramic: Some(true), + ..strict_image_builder() + }, + GenerateSplatRequestBuilder { + disable_recaption: Some(true), + ..strict_image_builder() + }, + ]; + for builder in cases { + assert!(build_fal_triposplat_state(builder).is_err()); + } + } + + #[test] + fn unsupported_options_are_dropped_under_lenient_strategies() { + let builder = GenerateSplatRequestBuilder { + prompt: Some("a cozy cabin".to_string()), + is_panoramic: Some(true), + disable_recaption: Some(true), + ..image_builder() + }; + assert!(build_fal_triposplat_state(builder).is_ok()); + } + + #[test] + fn is_panoramic_false_is_not_a_mismatch() { + let builder = GenerateSplatRequestBuilder { + is_panoramic: Some(false), + request_mismatch_mitigation_strategy: RequestMismatchMitigationStrategy::ErrorOut, + ..image_builder() + }; + assert!(build_fal_triposplat_state(builder).is_ok()); + } + } + + // ── Helpers ── + + fn base_builder() -> GenerateSplatRequestBuilder { + GenerateSplatRequestBuilder { + model: RouterSplatModel::TripoSplat, + provider: RouterProvider::Fal, + ..Default::default() + } + } + + fn image_builder() -> GenerateSplatRequestBuilder { + GenerateSplatRequestBuilder { + reference_images: Some(ImageListRef::Urls(vec![IMAGE_URL.to_string()])), + ..base_builder() + } + } + + fn strict_image_builder() -> GenerateSplatRequestBuilder { + GenerateSplatRequestBuilder { + request_mismatch_mitigation_strategy: RequestMismatchMitigationStrategy::ErrorOut, + ..image_builder() + } + } +} diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/cost.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/cost.rs new file mode 100644 index 0000000000..aa79c56426 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/cost.rs @@ -0,0 +1,59 @@ +use fal_client::requests::traits::fal_request_cost_calculator_trait::FalRequestCostCalculator; + +use crate::generate::generate_splat::providers::fal::triposplat::request::FalTripoSplatRequestState; +use crate::generate::generate_splat::splat_generation_cost_estimate::SplatGenerationCostEstimate; + +/// Cost math is owned by fal_client's `FalRequestCostCalculator` — the +/// router state just forwards the result so router cost ≡ fal_client cost +/// by construction. TripoSplat is flat 5¢. +#[derive(Clone, Debug)] +pub struct FalTripoSplatCostState { + pub cost_in_usd_cents: u64, +} + +impl FalTripoSplatCostState { + pub fn from_request(request: &FalTripoSplatRequestState) -> Self { + Self { + cost_in_usd_cents: request.request.calculate_cost_in_cents(), + } + } + + pub fn estimate_cost(&self) -> SplatGenerationCostEstimate { + SplatGenerationCostEstimate { + cost_in_credits: Some(self.cost_in_usd_cents), + cost_in_usd_cents: Some(self.cost_in_usd_cents), + is_free: false, + is_unlimited: false, + is_rate_limited: false, + has_watermark: false, + failures_are_refunded: None, + } + } +} + +#[cfg(test)] +mod tests { + use crate::api::image_list_ref::ImageListRef; + use crate::api::router_provider::RouterProvider; + use crate::api::router_splat_model::RouterSplatModel; + use crate::generate::generate_splat::generate_splat_request_builder::GenerateSplatRequestBuilder; + + #[test] + fn flat_five_cents() { + let builder = GenerateSplatRequestBuilder { + model: RouterSplatModel::TripoSplat, + provider: RouterProvider::Fal, + reference_images: Some(ImageListRef::Urls(vec![ + "https://example.com/object.png".to_string(), + ])), + ..Default::default() + }; + let cost = builder.build2() + .expect("build should succeed") + .estimate_cost() + .expect("estimate should succeed") + .cost_in_usd_cents + .expect("cost should be present"); + assert_eq!(cost, 5); + } +} diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/mod.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/mod.rs new file mode 100644 index 0000000000..98150bd014 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/mod.rs @@ -0,0 +1,3 @@ +pub mod build; +pub mod cost; +pub mod request; diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/request.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/request.rs new file mode 100644 index 0000000000..3e756b1f50 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/fal/triposplat/request.rs @@ -0,0 +1,62 @@ +use fal_client::requests::api::splat::image::triposplat_image_to_splat::api::TripoSplatImageToSplatRequest; + +use crate::client::router_fal_client::RouterFalClient; +use crate::errors::artcraft_router_error::ArtcraftRouterError; +use crate::generate::generate_splat::generate_splat_response::GenerateSplatResponse; +use crate::generate::generate_splat::providers::fal::request_common::send_fal_splat_request; + +#[derive(Clone, Debug)] +pub struct FalTripoSplatRequestState { + /// Final materialized request; ready to fire. + pub request: TripoSplatImageToSplatRequest, +} + +impl FalTripoSplatRequestState { + pub async fn send(&self, client: &RouterFalClient) -> Result { + send_fal_splat_request(&self.request, client).await + } +} + +#[cfg(test)] +mod tests { + use test_data::web::image_urls::ERNEST_SCARED_STUPID_IMAGE_URL; + + use crate::api::image_list_ref::ImageListRef; + use crate::api::router_provider::RouterProvider; + use crate::api::router_splat_model::RouterSplatModel; + use crate::generate::generate_splat::generate_splat_request_builder::GenerateSplatRequestBuilder; + use crate::generate::generate_splat::generate_splat_response::GenerateSplatResponse; + use crate::generate::generate_splat::splat_generation_draft_or_request::SplatGenerationDraftOrRequest; + use crate::test_helpers::get_fal_client; + + #[tokio::test] + #[ignore] // requires real API key, incurs cost + async fn send_triposplat_image_to_splat() { + let client = get_fal_client(); + + let builder = GenerateSplatRequestBuilder { + model: RouterSplatModel::TripoSplat, + provider: RouterProvider::Fal, + reference_images: Some(ImageListRef::Urls(vec![ + ERNEST_SCARED_STUPID_IMAGE_URL.to_string(), + ])), + ..Default::default() + }; + + let draft_or_request = builder.build2().expect("build2 should succeed"); + let request = match draft_or_request { + SplatGenerationDraftOrRequest::Request(r) => r, + _ => panic!("expected Request variant (Fal splat skips draft)"), + }; + + let response = request.send_request(&client).await.expect("send_request should succeed"); + println!("response: {:?}", response); + + let payload = match response { + GenerateSplatResponse::Fal(p) => p, + other => panic!("expected Fal payload, got {other:?}"), + }; + assert!(payload.request_id.is_some() || payload.gateway_request_id.is_some()); + assert_eq!(1, 2, "Inspect output above"); + } +} diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/mod.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/mod.rs index 980cdc0632..d24733020b 100644 --- a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/mod.rs +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/providers/mod.rs @@ -1,2 +1,3 @@ pub mod artcraft; +pub mod fal; pub mod worldlabs; diff --git a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/splat_generation_request.rs b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/splat_generation_request.rs index c62f463c99..769cff5d39 100644 --- a/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/splat_generation_request.rs +++ b/crates/api_clients/artcraft/artcraft_router/src/generate/generate_splat/splat_generation_request.rs @@ -10,6 +10,10 @@ use crate::generate::generate_splat::providers::artcraft::marble_1p1::cost::Artc use crate::generate::generate_splat::providers::artcraft::marble_1p1::request::ArtcraftMarble1p1RequestState; use crate::generate::generate_splat::providers::artcraft::marble_1p1_plus::cost::ArtcraftMarble1p1PlusCostState; use crate::generate::generate_splat::providers::artcraft::marble_1p1_plus::request::ArtcraftMarble1p1PlusRequestState; +use crate::generate::generate_splat::providers::artcraft::triposplat::cost::ArtcraftTripoSplatCostState; +use crate::generate::generate_splat::providers::artcraft::triposplat::request::ArtcraftTripoSplatRequestState; +use crate::generate::generate_splat::providers::fal::triposplat::cost::FalTripoSplatCostState; +use crate::generate::generate_splat::providers::fal::triposplat::request::FalTripoSplatRequestState; use crate::generate::generate_splat::providers::worldlabs::marble_1p0::cost::WorldLabsMarble1p0CostState; use crate::generate::generate_splat::providers::worldlabs::marble_1p0::request::WorldLabsMarble1p0RequestState; use crate::generate::generate_splat::providers::worldlabs::marble_1p0_draft::cost::WorldLabsMarble1p0DraftModelCostState; @@ -26,6 +30,8 @@ pub enum SplatGenerationRequest { ArtcraftMarble1p0Draft(ArtcraftMarble1p0DraftRequestState), ArtcraftMarble1p1(ArtcraftMarble1p1RequestState), ArtcraftMarble1p1Plus(ArtcraftMarble1p1PlusRequestState), + ArtcraftTripoSplat(ArtcraftTripoSplatRequestState), + FalTripoSplat(FalTripoSplatRequestState), WorldLabsMarble1p0(WorldLabsMarble1p0RequestState), WorldLabsMarble1p0Draft(WorldLabsMarble1p0DraftModelRequestState), WorldLabsMarble1p1(WorldLabsMarble1p1RequestState), @@ -40,6 +46,8 @@ impl SplatGenerationRequest { Self::ArtcraftMarble1p0Draft(_) => RouterProvider::Artcraft, Self::ArtcraftMarble1p1(_) => RouterProvider::Artcraft, Self::ArtcraftMarble1p1Plus(_) => RouterProvider::Artcraft, + Self::ArtcraftTripoSplat(_) => RouterProvider::Artcraft, + Self::FalTripoSplat(_) => RouterProvider::Fal, Self::WorldLabsMarble1p0(_) => RouterProvider::WorldLabs, Self::WorldLabsMarble1p0Draft(_) => RouterProvider::WorldLabs, Self::WorldLabsMarble1p1(_) => RouterProvider::WorldLabs, @@ -54,6 +62,8 @@ impl SplatGenerationRequest { SplatGenerationRequest::ArtcraftMarble1p0Draft(request) => Ok(ArtcraftMarble1p0DraftCostState::from_request(request).estimate_cost()), SplatGenerationRequest::ArtcraftMarble1p1(request) => Ok(ArtcraftMarble1p1CostState::from_request(request).estimate_cost()), SplatGenerationRequest::ArtcraftMarble1p1Plus(request) => Ok(ArtcraftMarble1p1PlusCostState::from_request(request).estimate_cost()), + SplatGenerationRequest::ArtcraftTripoSplat(request) => Ok(ArtcraftTripoSplatCostState::from_request(request).estimate_cost()), + SplatGenerationRequest::FalTripoSplat(request) => Ok(FalTripoSplatCostState::from_request(request).estimate_cost()), SplatGenerationRequest::WorldLabsMarble1p0(request) => Ok(WorldLabsMarble1p0CostState::from_request(request).estimate_cost()), SplatGenerationRequest::WorldLabsMarble1p0Draft(request) => Ok(WorldLabsMarble1p0DraftModelCostState::from_request(request).estimate_cost()), SplatGenerationRequest::WorldLabsMarble1p1(request) => Ok(WorldLabsMarble1p1CostState::from_request(request).estimate_cost()), @@ -81,6 +91,14 @@ impl SplatGenerationRequest { let client_ref = client.get_artcraft_client_ref()?; request.send(client_ref).await }, + SplatGenerationRequest::ArtcraftTripoSplat(request) => { + let client_ref = client.get_artcraft_client_ref()?; + request.send(client_ref).await + }, + SplatGenerationRequest::FalTripoSplat(request) => { + let client_ref = client.get_fal_client_ref()?; + request.send(client_ref).await + }, SplatGenerationRequest::WorldLabsMarble1p0(request) => { let client_ref = client.get_worldlabs_client_ref()?; request.send(client_ref).await diff --git a/crates/api_clients/fal_client/src/requests/api/mod.rs b/crates/api_clients/fal_client/src/requests/api/mod.rs index f3fac572fa..8272f0437d 100644 --- a/crates/api_clients/fal_client/src/requests/api/mod.rs +++ b/crates/api_clients/fal_client/src/requests/api/mod.rs @@ -1,4 +1,5 @@ pub mod audio; pub mod image; pub mod mesh; +pub mod splat; pub mod video; \ No newline at end of file diff --git a/crates/api_clients/fal_client/src/requests/api/splat/image/mod.rs b/crates/api_clients/fal_client/src/requests/api/splat/image/mod.rs new file mode 100644 index 0000000000..82f151561a --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/api/splat/image/mod.rs @@ -0,0 +1 @@ +pub mod triposplat_image_to_splat; diff --git a/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/api.rs b/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/api.rs new file mode 100644 index 0000000000..7e7b9ac982 --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/api.rs @@ -0,0 +1,160 @@ +use crate::error::fal_error_plus::FalErrorPlus; +use crate::requests::api::splat::image::triposplat_image_to_splat::raw_request::{ + TripoSplatImageToSplatInput, TripoSplatImageToSplatOutput, +}; +use crate::requests::traits::fal_endpoint_trait::FalEndpoint; + +/// TripoSplat image-to-Gaussian-splat: reconstructs a 3D Gaussian splat from +/// a single input image. +#[derive(Clone, Debug)] +pub struct TripoSplatImageToSplatRequest { + /// URL of the input image (required). + pub image_url: String, + + /// Number of Gaussians to generate. Range 32768-262144; + /// fal's default is 262144 when `None`. + pub num_gaussians: Option, + + /// Number of inference steps. Range 1-50; fal's default is 20 when `None`. + pub num_inference_steps: Option, + + /// Guidance scale. Range 0-10; fal's default is 3 when `None`. + pub guidance_scale: Option, + + /// Output file format. fal's default is `Ply` when `None`. + pub output_format: Option, + + /// Seed for reproducibility. fal picks a random seed when `None`. + pub seed: Option, + + /// fal's default is true when `None`. + pub enable_safety_checker: Option, +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum TripoSplatOutputFormat { + Ply, + Splat, +} + +impl TripoSplatOutputFormat { + pub(crate) fn to_str(&self) -> &'static str { + match self { + Self::Ply => "ply", + Self::Splat => "splat", + } + } +} + +impl FalEndpoint for TripoSplatImageToSplatRequest { + const ENDPOINT: &str = "tripo3d/triposplat"; + + type RawRequest = TripoSplatImageToSplatInput; + type RawResponse = TripoSplatImageToSplatOutput; + + fn to_raw_request(&self) -> Result { + Ok(Self::RawRequest { + image_url: self.image_url.clone(), + num_gaussians: self.num_gaussians, + num_inference_steps: self.num_inference_steps, + guidance_scale: self.guidance_scale, + output_format: self.output_format.map(|f| f.to_str().to_string()), + seed: self.seed, + enable_safety_checker: self.enable_safety_checker, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::traits::fal_endpoint_trait::FalEndpoint; + use errors::AnyhowResult; + use std::fs::read_to_string; + use test_data::web::image_urls::ERNEST_SCARED_STUPID_IMAGE_URL; + + mod serialization_tests { + use super::*; + + #[test] + fn minimal_request_serializes_image_url_only() { + let raw = base_request().to_raw_request().expect("to_raw_request"); + let json = serde_json::to_value(&raw).unwrap(); + let obj = json.as_object().unwrap(); + assert_eq!(obj.len(), 1, "only image_url should serialize: {json}"); + assert_eq!(json["image_url"], "https://example.com/input.png"); + } + + #[test] + fn output_format_wire_values() { + assert_eq!(TripoSplatOutputFormat::Ply.to_str(), "ply"); + assert_eq!(TripoSplatOutputFormat::Splat.to_str(), "splat"); + } + + #[test] + fn full_request_serializes_all_fields() { + let request = TripoSplatImageToSplatRequest { + num_gaussians: Some(131_072), + num_inference_steps: Some(30), + guidance_scale: Some(5.0), + output_format: Some(TripoSplatOutputFormat::Ply), + seed: Some(42), + enable_safety_checker: Some(false), + ..base_request() + }; + let raw = request.to_raw_request().expect("to_raw_request"); + let json = serde_json::to_value(&raw).unwrap(); + assert_eq!(json["num_gaussians"], 131_072); + assert_eq!(json["num_inference_steps"], 30); + assert_eq!(json["guidance_scale"], 5.0); + assert_eq!(json["output_format"], "ply"); + assert_eq!(json["seed"], 42); + assert_eq!(json["enable_safety_checker"], false); + } + } + + // ── Helpers ── + + fn base_request() -> TripoSplatImageToSplatRequest { + TripoSplatImageToSplatRequest { + image_url: "https://example.com/input.png".to_string(), + num_gaussians: None, + num_inference_steps: None, + guidance_scale: None, + output_format: None, + seed: None, + enable_safety_checker: None, + } + } + + #[tokio::test] + #[ignore] // manually run — fires a real API request and incurs cost + async fn test_image_to_splat_webhook() -> AnyhowResult<()> { + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + let api_key = FalApiKey::from_str(&secret); + let request = TripoSplatImageToSplatRequest { + image_url: ERNEST_SCARED_STUPID_IMAGE_URL.to_string(), + ..base_request() + }; + let result = request.send_webhook_request(&api_key, "https://example.com/webhook").await?; + println!("Webhook result: {:?}", result); + assert!(result.request_id.is_some() || result.gateway_request_id.is_some()); + Ok(()) + } + + #[tokio::test] + #[ignore] // manually run — fires a real API request and incurs cost + async fn test_image_to_splat_queue() -> AnyhowResult<()> { + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + let api_key = FalApiKey::from_str(&secret); + let request = TripoSplatImageToSplatRequest { + image_url: ERNEST_SCARED_STUPID_IMAGE_URL.to_string(), + ..base_request() + }; + let result = request.send_queue_request(&api_key).await?; + println!("Queue result — request_id: {}", result.request_id); + assert!(!result.request_id.is_empty()); + Ok(()) + } +} diff --git a/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/cost.rs b/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/cost.rs new file mode 100644 index 0000000000..8baaed8774 --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/cost.rs @@ -0,0 +1,47 @@ +use crate::requests::api::splat::image::triposplat_image_to_splat::api::TripoSplatImageToSplatRequest; +use crate::requests::traits::fal_request_cost_calculator_trait::{FalRequestCostCalculator, UsdCents}; + +impl FalRequestCostCalculator for TripoSplatImageToSplatRequest { + fn calculate_cost_in_cents(&self) -> UsdCents { + // Pricing (see https://fal.ai/models/tripo3d/triposplat): + // "Price: $0.05 per generations" + 5 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::requests::api::splat::image::triposplat_image_to_splat::api::TripoSplatOutputFormat; + + fn base_request() -> TripoSplatImageToSplatRequest { + TripoSplatImageToSplatRequest { + image_url: "https://example.com/input.png".to_string(), + num_gaussians: None, + num_inference_steps: None, + guidance_scale: None, + output_format: None, + seed: None, + enable_safety_checker: None, + } + } + + #[test] + fn flat_five_cents() { + assert_eq!(base_request().calculate_cost_in_cents(), 5); + } + + #[test] + fn options_do_not_change_cost() { + let request = TripoSplatImageToSplatRequest { + num_gaussians: Some(32_768), + num_inference_steps: Some(50), + guidance_scale: Some(10.0), + output_format: Some(TripoSplatOutputFormat::Splat), + seed: Some(7), + enable_safety_checker: Some(false), + ..base_request() + }; + assert_eq!(request.calculate_cost_in_cents(), 5); + } +} diff --git a/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/mod.rs b/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/mod.rs new file mode 100644 index 0000000000..3bfb6c86db --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/mod.rs @@ -0,0 +1,3 @@ +pub mod api; +pub mod cost; +pub mod raw_request; diff --git a/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/raw_request.rs b/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/raw_request.rs new file mode 100644 index 0000000000..906bb24e71 --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/api/splat/image/triposplat_image_to_splat/raw_request.rs @@ -0,0 +1,37 @@ +use serde::{Deserialize, Serialize}; + +/// Over-the-wire input shape for `tripo3d/triposplat` (image-to-Gaussian-splat). +/// fal's schema: +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct TripoSplatImageToSplatInput { + /// URL of the input image. + pub image_url: String, + + /// Number of Gaussians to generate. Range 32768-262144. + /// fal default: 262144. + #[serde(skip_serializing_if = "Option::is_none")] + pub num_gaussians: Option, + + /// Number of inference steps. Range 1-50. fal default: 20. + #[serde(skip_serializing_if = "Option::is_none")] + pub num_inference_steps: Option, + + /// Guidance scale. Range 0-10. fal default: 3. + #[serde(skip_serializing_if = "Option::is_none")] + pub guidance_scale: Option, + + /// Options: "ply", "splat". fal default: "ply". + #[serde(skip_serializing_if = "Option::is_none")] + pub output_format: Option, + + /// Seed for reproducibility. fal default: random. + #[serde(skip_serializing_if = "Option::is_none")] + pub seed: Option, + + /// fal default: true. + #[serde(skip_serializing_if = "Option::is_none")] + pub enable_safety_checker: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct TripoSplatImageToSplatOutput {} diff --git a/crates/api_clients/fal_client/src/requests/api/splat/mod.rs b/crates/api_clients/fal_client/src/requests/api/splat/mod.rs new file mode 100644 index 0000000000..14995d4228 --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/api/splat/mod.rs @@ -0,0 +1 @@ +pub mod image; diff --git a/crates/lib/files/mimetypes/src/custom/custom_infer.rs b/crates/lib/files/mimetypes/src/custom/custom_infer.rs index f196f474fb..97d805ec1a 100644 --- a/crates/lib/files/mimetypes/src/custom/custom_infer.rs +++ b/crates/lib/files/mimetypes/src/custom/custom_infer.rs @@ -10,6 +10,7 @@ pub (crate) static CUSTOM_INFER: LazyLock = LazyLock::new(|| { fn add_custom_types(infer: &mut Infer) { infer.add("model/gltf-binary", "glb", gltf_matcher); infer.add("model/fbx", "fbx", fbx_matcher); + infer.add("model/ply", "ply", ply_matcher); } fn gltf_matcher(buf: &[u8]) -> bool { @@ -18,6 +19,14 @@ fn gltf_matcher(buf: &[u8]) -> bool { buf.len() >= 4 && buf[0] == 103 && buf[1] == 108 && buf[2] == 84 && buf[3] == 70 } +fn ply_matcher(buf: &[u8]) -> bool { + // PLY files (ASCII and binary alike) start with the line "ply". + // https://paulbourke.net/dataformats/ply/ + buf.len() >= 4 + && &buf[..3] == b"ply" + && (buf[3] == b'\n' || buf[3] == b'\r') +} + fn fbx_matcher(buf: &[u8]) -> bool { // Binary FBX magic: "Kaydara FBX Binary " (two trailing spaces). // https://code.blender.org/2013/08/fbx-binary-file-format-specification/ @@ -47,6 +56,19 @@ mod tests { assert_eq!(maybe_type.mime_type(), "model/fbx"); } + #[test] + fn test_ply_magic() { + let bytes = b"ply\nformat binary_little_endian 1.0\n".to_vec(); + let maybe_type = CUSTOM_INFER.get(&bytes).unwrap(); + assert_eq!(maybe_type.mime_type(), "model/ply"); + } + + #[test] + fn test_ply_prefix_without_newline_is_not_detected() { + let bytes = b"plywood texture".to_vec(); + assert!(CUSTOM_INFER.get(&bytes).is_none()); + } + #[test] fn test_ascii_fbx_is_not_detected() { // ASCII FBX files have no magic bytes. diff --git a/crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs b/crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs index 9e4f016467..2aa1825d86 100644 --- a/crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs +++ b/crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs @@ -17,6 +17,7 @@ pub enum FileExtension { Obj, Ogg, Opus, + Ply, Png, Wav, Webm, @@ -37,6 +38,7 @@ impl FileExtension { "audio/mpeg" => Some(FileExtension::Mp3), "video/mp4" => Some(FileExtension::Mp4), "model/obj" => Some(FileExtension::Obj), + "model/ply" => Some(FileExtension::Ply), "audio/ogg" => Some(FileExtension::Ogg), "audio/opus" => Some(FileExtension::Opus), "image/png" => Some(FileExtension::Png), @@ -69,6 +71,7 @@ impl FileExtension { FileExtension::Obj => (".obj", "obj"), FileExtension::Ogg => (".ogg", "ogg"), FileExtension::Opus => (".opus", "opus"), + FileExtension::Ply => (".ply", "ply"), FileExtension::Png => (".png", "png"), FileExtension::Wav => (".wav", "wav"), FileExtension::Webm => (".webm", "webm"), diff --git a/crates/schema/database/mysql_queries/src/queries/users/user_bookmarks/list_user_bookmarks_result.rs b/crates/schema/database/mysql_queries/src/queries/users/user_bookmarks/list_user_bookmarks_result.rs index 690ae8f6f6..90a5e44082 100644 --- a/crates/schema/database/mysql_queries/src/queries/users/user_bookmarks/list_user_bookmarks_result.rs +++ b/crates/schema/database/mysql_queries/src/queries/users/user_bookmarks/list_user_bookmarks_result.rs @@ -171,6 +171,7 @@ impl RawUserBookmarkRecord { MediaFileType::Bvh => Some("3d file".to_string()), MediaFileType::Fbx => Some("3d file".to_string()), MediaFileType::Obj => Some("3d file".to_string()), + MediaFileType::Ply => Some("3d splat".to_string()), MediaFileType::Glb => Some("3d file".to_string()), MediaFileType::Gltf => Some("3d file".to_string()), MediaFileType::Spz => Some("3d splat".to_string()), diff --git a/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index 01cdaee7b0..ff7f50e9d2 100644 --- a/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/schema/public/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -279,6 +279,8 @@ pub enum InferenceModelType { Marble1p1, #[serde(rename = "marble_1p1_plus")] Marble1p1Plus, + #[serde(rename = "triposplat")] + TripoSplat, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -411,6 +413,7 @@ impl InferenceModelType { Self::Marble1p0Draft => "marble_1p0_draft", Self::Marble1p1 => "marble_1p1", Self::Marble1p1Plus => "marble_1p1_plus", + Self::TripoSplat => "triposplat", } } @@ -538,6 +541,7 @@ impl InferenceModelType { "marble_1p0_draft" => Ok(Self::Marble1p0Draft), "marble_1p1" => Ok(Self::Marble1p1), "marble_1p1_plus" => Ok(Self::Marble1p1Plus), + "triposplat" => Ok(Self::TripoSplat), _ => Err(format!("invalid value: {:?}", value)), } @@ -669,6 +673,7 @@ impl InferenceModelType { Self::Marble1p0Draft, Self::Marble1p1, Self::Marble1p1Plus, + Self::TripoSplat, ]) } @@ -783,6 +788,7 @@ impl InferenceModelType { CommonModelType::Marble1p0Draft => Self::Marble1p0Draft, CommonModelType::Marble1p1 => Self::Marble1p1, CommonModelType::Marble1p1Plus => Self::Marble1p1Plus, + CommonModelType::TripoSplat => Self::TripoSplat, } } } @@ -918,6 +924,7 @@ mod tests { assert_serialization(InferenceModelType::Marble1p0Draft, "marble_1p0_draft"); assert_serialization(InferenceModelType::Marble1p1, "marble_1p1"); assert_serialization(InferenceModelType::Marble1p1Plus, "marble_1p1_plus"); + assert_serialization(InferenceModelType::TripoSplat, "triposplat"); } #[test] @@ -1044,6 +1051,7 @@ mod tests { assert_eq!(InferenceModelType::Marble1p0Draft.to_str(), "marble_1p0_draft"); assert_eq!(InferenceModelType::Marble1p1.to_str(), "marble_1p1"); assert_eq!(InferenceModelType::Marble1p1Plus.to_str(), "marble_1p1_plus"); + assert_eq!(InferenceModelType::TripoSplat.to_str(), "triposplat"); } #[test] @@ -1168,6 +1176,7 @@ mod tests { assert_eq!(InferenceModelType::from_str("marble_1p0_draft").unwrap(), InferenceModelType::Marble1p0Draft); assert_eq!(InferenceModelType::from_str("marble_1p1").unwrap(), InferenceModelType::Marble1p1); assert_eq!(InferenceModelType::from_str("marble_1p1_plus").unwrap(), InferenceModelType::Marble1p1Plus); + assert_eq!(InferenceModelType::from_str("triposplat").unwrap(), InferenceModelType::TripoSplat); } #[test] diff --git a/crates/schema/public/enums/src/by_table/media_files/media_file_type.rs b/crates/schema/public/enums/src/by_table/media_files/media_file_type.rs index 7764a5d47f..daa1dfac6d 100644 --- a/crates/schema/public/enums/src/by_table/media_files/media_file_type.rs +++ b/crates/schema/public/enums/src/by_table/media_files/media_file_type.rs @@ -40,6 +40,9 @@ pub enum MediaFileType { /// Wavefront OBJ mesh files Obj, + /// PLY (polygon file format) files; also used for Gaussian splats. + Ply, + /// glTF binary files (for Bevy) Glb, @@ -178,7 +181,8 @@ impl MediaFileType { | Self::Pmd | Self::Pmx => MediaFileClass::Mesh, - Self::Spz => MediaFileClass::Splat, + Self::Spz + | Self::Ply => MediaFileClass::Splat, // Animation and auxiliary data formats with no dedicated class yet. #[allow(deprecated)] @@ -223,6 +227,7 @@ impl MediaFileType { "model/fbx" => Some(Self::Fbx), "application/fbx" => Some(Self::Fbx), "model/obj" => Some(Self::Obj), + "model/ply" => Some(Self::Ply), _ => None, } } @@ -282,6 +287,7 @@ impl MediaFileType { Self::Bvh => "bvh", Self::Fbx => "fbx", Self::Obj => "obj", + Self::Ply => "ply", Self::Glb => "glb", Self::Gltf => "gltf", Self::Spz => "spz", @@ -317,6 +323,7 @@ impl MediaFileType { "bvh" => Ok(Self::Bvh), "fbx" => Ok(Self::Fbx), "obj" => Ok(Self::Obj), + "ply" => Ok(Self::Ply), "glb" => Ok(Self::Glb), "gltf" => Ok(Self::Gltf), "spz" => Ok(Self::Spz), @@ -355,6 +362,7 @@ impl MediaFileType { Self::Bvh, Self::Fbx, Self::Obj, + Self::Ply, Self::Glb, Self::Gltf, Self::Spz, @@ -423,6 +431,7 @@ mod tests { assert_serialization(MediaFileType::Bvh, "bvh"); assert_serialization(MediaFileType::Fbx, "fbx"); assert_serialization(MediaFileType::Obj, "obj"); + assert_serialization(MediaFileType::Ply, "ply"); assert_serialization(MediaFileType::Glb, "glb"); assert_serialization(MediaFileType::Gltf, "gltf"); assert_serialization(MediaFileType::Spz, "spz"); @@ -531,6 +540,7 @@ mod tests { assert_eq!(MediaFileType::Glb.to_media_class(), MediaFileClass::Mesh); assert_eq!(MediaFileType::Gltf.to_media_class(), MediaFileClass::Mesh); assert_eq!(MediaFileType::Spz.to_media_class(), MediaFileClass::Splat); + assert_eq!(MediaFileType::Ply.to_media_class(), MediaFileClass::Splat); assert_eq!(MediaFileType::SceneRon.to_media_class(), MediaFileClass::Project); assert_eq!(MediaFileType::SceneJson.to_media_class(), MediaFileClass::Project); assert_eq!(MediaFileType::Pmd.to_media_class(), MediaFileClass::Mesh); @@ -695,13 +705,14 @@ mod tests { #[test] fn all_variants() { let mut variants = MediaFileType::all_variants(); - assert_eq!(variants.len(), 30); + assert_eq!(variants.len(), 31); assert_eq!(variants.pop_first(), Some(MediaFileType::Audio)); assert_eq!(variants.pop_first(), Some(MediaFileType::Image)); assert_eq!(variants.pop_first(), Some(MediaFileType::Video)); assert_eq!(variants.pop_first(), Some(MediaFileType::Bvh)); assert_eq!(variants.pop_first(), Some(MediaFileType::Fbx)); assert_eq!(variants.pop_first(), Some(MediaFileType::Obj)); + assert_eq!(variants.pop_first(), Some(MediaFileType::Ply)); assert_eq!(variants.pop_first(), Some(MediaFileType::Glb)); assert_eq!(variants.pop_first(), Some(MediaFileType::Gltf)); assert_eq!(variants.pop_first(), Some(MediaFileType::Spz)); diff --git a/crates/schema/public/enums/src/common/generation/common_model_type.rs b/crates/schema/public/enums/src/common/generation/common_model_type.rs index aaa600dac0..112405af61 100644 --- a/crates/schema/public/enums/src/common/generation/common_model_type.rs +++ b/crates/schema/public/enums/src/common/generation/common_model_type.rs @@ -248,6 +248,8 @@ pub enum CommonModelType { Marble1p1, #[serde(rename = "marble_1p1_plus")] Marble1p1Plus, + #[serde(rename = "triposplat")] + TripoSplat, } impl_enum_display_and_debug_using_to_str!(CommonModelType); @@ -364,6 +366,7 @@ impl CommonModelType { Self::Marble1p0Draft => "marble_1p0_draft", Self::Marble1p1 => "marble_1p1", Self::Marble1p1Plus => "marble_1p1_plus", + Self::TripoSplat => "triposplat", } } @@ -474,6 +477,7 @@ impl CommonModelType { "marble_1p0_draft" => Ok(Self::Marble1p0Draft), "marble_1p1" => Ok(Self::Marble1p1), "marble_1p1_plus" => Ok(Self::Marble1p1Plus), + "triposplat" => Ok(Self::TripoSplat), _ => Err(format!("invalid model_type: {:?}", job_status)), } @@ -588,6 +592,7 @@ impl CommonModelType { Self::Marble1p0Draft, Self::Marble1p1, Self::Marble1p1Plus, + Self::TripoSplat, ]) } @@ -699,6 +704,7 @@ impl CommonModelType { Self::Marble1p0Draft => CommonModelClass::DimensionalSplat, Self::Marble1p1 => CommonModelClass::DimensionalSplat, Self::Marble1p1Plus => CommonModelClass::DimensionalSplat, + Self::TripoSplat => CommonModelClass::DimensionalSplat, } } } @@ -815,6 +821,7 @@ mod tests { assert_serialization(CommonModelType::Marble1p0Draft, "marble_1p0_draft"); assert_serialization(CommonModelType::Marble1p1, "marble_1p1"); assert_serialization(CommonModelType::Marble1p1Plus, "marble_1p1_plus"); + assert_serialization(CommonModelType::TripoSplat, "triposplat"); } #[test] @@ -923,6 +930,7 @@ mod tests { assert_eq!(CommonModelType::Marble1p0Draft.to_str(), "marble_1p0_draft"); assert_eq!(CommonModelType::Marble1p1.to_str(), "marble_1p1"); assert_eq!(CommonModelType::Marble1p1Plus.to_str(), "marble_1p1_plus"); + assert_eq!(CommonModelType::TripoSplat.to_str(), "triposplat"); } #[test] @@ -1028,12 +1036,13 @@ mod tests { assert_eq!(CommonModelType::from_str("marble_1p0_draft").unwrap(), CommonModelType::Marble1p0Draft); assert_eq!(CommonModelType::from_str("marble_1p1").unwrap(), CommonModelType::Marble1p1); assert_eq!(CommonModelType::from_str("marble_1p1_plus").unwrap(), CommonModelType::Marble1p1Plus); + assert_eq!(CommonModelType::from_str("triposplat").unwrap(), CommonModelType::TripoSplat); } #[test] fn all_variants() { let mut variants = CommonModelType::all_variants(); - assert_eq!(variants.len(), 95); + assert_eq!(variants.len(), 96); // Image models assert_eq!(variants.pop_first(), Some(CommonModelType::Flux1Dev)); assert_eq!(variants.pop_first(), Some(CommonModelType::Flux1Schnell)); @@ -1134,6 +1143,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(CommonModelType::Marble1p0Draft)); assert_eq!(variants.pop_first(), Some(CommonModelType::Marble1p1)); assert_eq!(variants.pop_first(), Some(CommonModelType::Marble1p1Plus)); + assert_eq!(variants.pop_first(), Some(CommonModelType::TripoSplat)); assert_eq!(variants.pop_first(), None); } diff --git a/crates/schema/public/enums/src/common/generation/common_splat_model.rs b/crates/schema/public/enums/src/common/generation/common_splat_model.rs index 989e38d90a..3bf50e72bd 100644 --- a/crates/schema/public/enums/src/common/generation/common_splat_model.rs +++ b/crates/schema/public/enums/src/common/generation/common_splat_model.rs @@ -21,6 +21,10 @@ pub enum CommonSplatModel { #[serde(rename = "marble_1p1_plus")] Marble1p1Plus, + + /// TripoSplat: single-image to 3D Gaussian splat (via Fal). + #[serde(rename = "triposplat")] + TripoSplat, } impl CommonSplatModel { @@ -33,6 +37,7 @@ impl CommonSplatModel { Self::Marble1p0Draft => CommonModelType::Marble1p0Draft, Self::Marble1p1 => CommonModelType::Marble1p1, Self::Marble1p1Plus => CommonModelType::Marble1p1Plus, + Self::TripoSplat => CommonModelType::TripoSplat, } } } @@ -51,6 +56,7 @@ mod tests { assert_serialization(CommonSplatModel::Marble1p0Draft, "marble_1p0_draft"); assert_serialization(CommonSplatModel::Marble1p1, "marble_1p1"); assert_serialization(CommonSplatModel::Marble1p1Plus, "marble_1p1_plus"); + assert_serialization(CommonSplatModel::TripoSplat, "triposplat"); } #[test] @@ -62,6 +68,7 @@ mod tests { ("marble_1p0_draft", CommonSplatModel::Marble1p0Draft), ("marble_1p1", CommonSplatModel::Marble1p1), ("marble_1p1_plus", CommonSplatModel::Marble1p1Plus), + ("triposplat", CommonSplatModel::TripoSplat), ]; for (json_str, expected) in cases { let json = format!("\"{}\"", json_str); @@ -80,6 +87,7 @@ mod tests { CommonSplatModel::Marble1p0Draft, CommonSplatModel::Marble1p1, CommonSplatModel::Marble1p1Plus, + CommonSplatModel::TripoSplat, ]; for variant in all { let json = serde_json::to_string(&variant).unwrap(); @@ -97,6 +105,7 @@ mod tests { (CommonSplatModel::Marble1p0Draft, CommonModelType::Marble1p0Draft), (CommonSplatModel::Marble1p1, CommonModelType::Marble1p1), (CommonSplatModel::Marble1p1Plus, CommonModelType::Marble1p1Plus), + (CommonSplatModel::TripoSplat, CommonModelType::TripoSplat), ]; for (splat_model, expected) in models { assert_eq!(splat_model.to_common_model_type(), expected); diff --git a/crates/service/web/storyteller_web/src/configs/omni_gen/splat_models.rs b/crates/service/web/storyteller_web/src/configs/omni_gen/splat_models.rs index b783af876c..3e0b2cd0f7 100644 --- a/crates/service/web/storyteller_web/src/configs/omni_gen/splat_models.rs +++ b/crates/service/web/storyteller_web/src/configs/omni_gen/splat_models.rs @@ -28,6 +28,7 @@ fn build_omni_gen_splat_models() -> Vec { marble_model(CommonSplatModel::Marble1p0Draft, "Marble 1.0 Draft"), marble_model(CommonSplatModel::Marble1p1, "Marble 1.1"), marble_model(CommonSplatModel::Marble1p1Plus, "Marble 1.1 Plus"), + triposplat_model(), ] } @@ -53,12 +54,31 @@ fn build_omni_gen_splat_model_providers() -> Vec OmniGenSplatModelDetails { + OmniGenSplatModelDetails { + model: CommonSplatModel::TripoSplat, + model_creator: Some(ModelCreator::Tripo), + full_name: Some("TripoSplat".to_string()), + extra_info: Some("Reconstructs a 3D Gaussian splat from a single image".to_string()), + extra_info_short: Some("Image to splat".to_string()), + image_references_supported: Some(true), + image_references_max: Some(1), + ..Default::default() + } +} + /// All Marble models share the same capability surface: text prompt, image /// references (up to 4, multi-view), a reference video, 360-degree panorama /// input, and the "disable recaption" toggle. diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/helpers/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/helpers/mod.rs index b3a95eb972..7c3ebd85ff 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/helpers/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod hydrate_router_request; pub mod pipeline_result; +pub mod resolve_media_tokens_to_urls; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/helpers/resolve_media_tokens_to_urls.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/helpers/resolve_media_tokens_to_urls.rs new file mode 100644 index 0000000000..47bbabf5da --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/helpers/resolve_media_tokens_to_urls.rs @@ -0,0 +1,38 @@ +use std::collections::HashMap; + +use artcraft_router::api::image_list_ref::ImageListRef; +use artcraft_router::api::video_ref::VideoRef; +use artcraft_router::generate::generate_splat::generate_splat_request_builder::GenerateSplatRequestBuilder; +use tokens::tokens::media_files::MediaFileToken; + +/// For providers that take URLs directly (Fal), swap +/// `MediaFileToken(s)` → `Url(s)` using the resolved map. +/// +/// NB: World Labs requests must keep their tokens — their draft/finalize +/// step maps tokens to URLs itself — so only call this on the Fal path. +pub fn resolve_media_tokens_to_urls( + builder: &mut GenerateSplatRequestBuilder, + url_map: Option<&HashMap>, +) { + let map = match url_map { + Some(m) => m, + None => return, + }; + + // reference_images + if let Some(ImageListRef::MediaFileTokens(ref tokens)) = builder.reference_images { + let urls: Vec = tokens.iter() + .filter_map(|t| map.get(t).cloned()) + .collect(); + if !urls.is_empty() { + builder.reference_images = Some(ImageListRef::Urls(urls)); + } + } + + // reference_video + if let Some(VideoRef::MediaFileToken(ref token)) = builder.reference_video { + if let Some(url) = map.get(token) { + builder.reference_video = Some(VideoRef::Url(url.clone())); + } + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/insert_db_job/insert_fal_job.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/insert_db_job/insert_fal_job.rs new file mode 100644 index 0000000000..061b63e147 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/insert_db_job/insert_fal_job.rs @@ -0,0 +1,56 @@ +use log::warn; + +use enums::common::visibility::Visibility; +use mysql_queries::queries::generic_inference::api_providers::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::api_providers::fal::insert_generic_inference_job_for_fal_queue_with_apriori_job_token::{ + insert_generic_inference_job_for_fal_queue_with_apriori_job_token, + InsertGenericInferenceForFalWithAprioriJobTokenArgs, +}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::endpoints::omni_gen::generate::video::insert_db_job::shared_job_args::SharedJobArgs; + +pub struct InsertFalJobArgs<'a, 'tx> { + pub external_job_id: &'a str, + pub shared: SharedJobArgs<'a, 'tx>, +} + +pub async fn insert_fal_job(args: InsertFalJobArgs<'_, '_>) -> Result { + let InsertFalJobArgs { + external_job_id, + shared, + } = args; + + let db_result = insert_generic_inference_job_for_fal_queue_with_apriori_job_token( + InsertGenericInferenceForFalWithAprioriJobTokenArgs { + apriori_job_token: shared.apriori_job_token, + uuid_idempotency_token: shared.idempotency_token, + maybe_external_third_party_id: external_job_id, + fal_category: FalCategory::ObjectGeneration, + maybe_model_type: shared.maybe_model_type, + maybe_prompt_token: shared.maybe_prompt_token, + maybe_creator_user_token: Some(shared.user_token), + maybe_avt_token: shared.maybe_avt_token, + creator_ip_address: shared.ip_address, + creator_set_visibility: Visibility::Public, + maybe_platform_type: shared.maybe_platform_type, + maybe_cost_estimates: shared.maybe_cost_estimates, + maybe_debug_log_event_token: shared.maybe_debug_log_event_token, + mysql_executor: &mut **shared.transaction, + maybe_inference_args: None, + starting_job_status_override: None, + maybe_frontend_failure_category: None, + maybe_failure_reason: None, + phantom: Default::default(), + } + ).await; + + match db_result { + Ok(token) => Ok(token), + Err(err) => { + warn!("Error inserting fal splat inference job: {:?}", err); + Err(CommonWebError::from_error(err)) + } + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/insert_db_job/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/insert_db_job/mod.rs index bc6a24943e..4c73c1042f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/insert_db_job/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/insert_db_job/mod.rs @@ -1 +1,2 @@ +pub mod insert_fal_job; pub mod insert_worldlabs_job; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/omni_gen_splat_generate_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/omni_gen_splat_generate_handler.rs index 3c69c9bd89..1a81708a26 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/omni_gen_splat_generate_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/omni_gen_splat_generate_handler.rs @@ -31,6 +31,7 @@ use crate::http_server::common_responses::common_web_error::CommonWebError; use crate::http_server::endpoints::generate::common::generation_debug_logs::GenerationDebugLogContext; use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test; use crate::http_server::endpoints::omni_gen::generate::splat::helpers::hydrate_router_request::hydrate_to_router_request; +use crate::http_server::endpoints::omni_gen::generate::splat::insert_db_job::insert_fal_job::{insert_fal_job, InsertFalJobArgs}; use crate::http_server::endpoints::omni_gen::generate::splat::insert_db_job::insert_worldlabs_job::{insert_worldlabs_job, InsertWorldlabsJobArgs}; use crate::http_server::endpoints::omni_gen::generate::splat::pipeline_v2::run_pipeline_v2::{run_pipeline_v2, RunPipelineV2Args}; use crate::http_server::endpoints::omni_gen::generate::video::insert_db_job::shared_job_args::SharedJobArgs; @@ -312,6 +313,35 @@ pub async fn omni_gen_splat_generate_handler( vec![token], ) } + GenerateSplatResponse::Fal(payload) => { + let external_id = payload.request_id.as_deref().ok_or_else(|| { + error!("Fal splat generation response missing request_id"); + CommonWebError::server_error_with_message("Fal generation response missing request_id") + })?; + info!("Inserting fal splat job with token: {:?}", pipeline_result.billing.apriori_job_token); + + let token = insert_fal_job(InsertFalJobArgs { + external_job_id: external_id, + shared: SharedJobArgs { + apriori_job_token: &pipeline_result.billing.apriori_job_token, + idempotency_token: &idempotency_token, + user_token, + maybe_avt_token: maybe_avt_token.as_ref(), + maybe_model_type: maybe_prompt_model_type, + maybe_prompt_token: prompt_token.as_ref(), + maybe_debug_log_event_token: Some(&debug_log_event_token), + maybe_platform_type, + maybe_cost_estimates: Some(pipeline_result.cost_estimates), + ip_address: &ip_address, + transaction: &mut transaction, + }, + }).await?; + + ( + token.clone(), + vec![token], + ) + } GenerateSplatResponse::Artcraft(payload) => { ( payload.inference_job_token.clone(), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/pipeline_v2/run_pipeline_v2.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/pipeline_v2/run_pipeline_v2.rs index 008fb931a0..720aa09933 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/pipeline_v2/run_pipeline_v2.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/splat/pipeline_v2/run_pipeline_v2.rs @@ -4,6 +4,7 @@ use std::convert::TryFrom; use log::{error, info, warn}; use artcraft_router::api::router_provider::RouterProvider; +use artcraft_router::api::router_splat_model::RouterSplatModel; use artcraft_router::generate::generate_splat::generate_splat_request_builder::GenerateSplatRequestBuilder; use artcraft_router::generate::generate_splat::generate_splat_response::GenerateSplatResponse; use artcraft_router::generate::generate_splat::splat_generation_draft_context::SplatGenerationDraftContext; @@ -18,6 +19,7 @@ use crate::http_server::endpoints::generate::common::generation_debug_logs::{ insert_provider_request_debug_log, provider_request_debug_log_type, GenerationDebugLogContext, }; use crate::http_server::endpoints::omni_gen::generate::splat::helpers::pipeline_result::PipelineResult; +use crate::http_server::endpoints::omni_gen::generate::splat::helpers::resolve_media_tokens_to_urls::resolve_media_tokens_to_urls; use crate::http_server::endpoints::omni_gen::generate::video::helpers::bill_wallet::bill_wallet; use crate::http_server::endpoints::omni_gen::generate::video::helpers::build_router_client::build_router_client; use crate::http_server::endpoints::omni_gen::generate::video::kinovi_account::KinoviAccount; @@ -52,16 +54,24 @@ pub async fn run_pipeline_v2(args: RunPipelineV2Args<'_>) -> Result RouterProvider::Fal, + _ => RouterProvider::WorldLabs, + }; // 1. Build execution request. // World Labs media inputs stay token-typed here — requests with media // return a Draft whose finalize step maps tokens to Artcraft URLs via // the draft context (mirroring the audio pipeline's Kinovi drafts). + // Fal takes media URLs directly, so its tokens resolve before building. let mut exec_builder = router_builder.clone(); exec_builder.provider = provider; + if matches!(provider, RouterProvider::Fal) { + resolve_media_tokens_to_urls(&mut exec_builder, media_file_to_url_map.as_ref()); + } + let draft_or_request = exec_builder.build2() .map_err(|e| { warn!("Failed to build2 for splat v2 pipeline: {}", e); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/splat/validate_splat_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/splat/validate_splat_request.rs index 2cd330c086..926a9fd1da 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/splat/validate_splat_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/splat/validate_splat_request.rs @@ -1,4 +1,5 @@ use artcraft_api_defs::omni_gen::cost_and_generate_requests::omni_gen_splat_cost_and_generate_request::OmniGenSplatCostAndGenerateRequest; +use enums::common::generation::common_splat_model::CommonSplatModel; use crate::http_server::common_responses::common_web_error::CommonWebError; @@ -11,8 +12,8 @@ const MAX_IMAGE_REFERENCES: usize = 4; /// NB: The cost endpoint deliberately does NOT call this — the UI polls for /// a price while the user is still composing the request. /// -/// Model presence is enforced later during hydration; the input-shape rules -/// below apply uniformly to all Marble models. +/// Model presence is enforced later during hydration. The Marble family +/// shares one set of input-shape rules; TripoSplat is image-only. pub fn validate_splat_request( request: &OmniGenSplatCostAndGenerateRequest, ) -> Result<(), CommonWebError> { @@ -20,6 +21,27 @@ pub fn validate_splat_request( let has_video = request.reference_video_media_token.is_some(); let has_prompt = request.prompt.as_deref().is_some_and(|p| !p.trim().is_empty()); + // TripoSplat reconstructs a splat from exactly one image; it takes no + // prompt, video, or panorama flag. + if request.model == Some(CommonSplatModel::TripoSplat) { + if image_reference_count != 1 { + return Err(bad_input( + "model triposplat requires exactly one reference image".to_string(), + )); + } + if has_video { + return Err(bad_input( + "model triposplat does not support a reference video".to_string(), + )); + } + if request.is_panoramic == Some(true) { + return Err(bad_input( + "model triposplat does not support panoramic input".to_string(), + )); + } + return Ok(()); + } + if !has_prompt && image_reference_count == 0 && !has_video { return Err(bad_input( "at least one of prompt, reference images, or a reference video must be provided".to_string(), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_model_mesh_payload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_model_mesh_payload.rs index ae12d619ed..270f82893f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_model_mesh_payload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_model_mesh_payload.rs @@ -6,6 +6,7 @@ use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path: use enums::by_table::media_files::media_file_class::MediaFileClass; use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; use enums::by_table::media_files::media_file_type::MediaFileType; use fal_client::webhook_payload::hydrated::hydrated_webhook_contents::{ModelMeshData, ModelObjData}; use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; @@ -149,6 +150,12 @@ pub(crate) async fn upload_mesh_file( CommonWebError::from_anyhow_error(err) })?; + // PLY results are Gaussian splats (e.g. TripoSplat), not meshes: store + // them the way the World Labs splat pipeline does — tagged as world + // generation, with no `Object` engine category. No fal mesh endpoint + // returns PLY, so the file type is a reliable discriminator. + let is_gaussian_splat = matches!(media_file_type, MediaFileType::Ply); + let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(&extension_with_period)); info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); @@ -163,14 +170,13 @@ pub(crate) async fn upload_mesh_file( CommonWebError::from_anyhow_error(err) })?; - let media_token = MediaFileInsertBuilder::new() + let mut insert_builder = MediaFileInsertBuilder::new() .checksum_sha2(&file_hash) .creator_ip_address(&job.creator_ip_address) .file_size_bytes(file_size_bytes as u64) .maybe_batch_generation_token(maybe_batch_token) .maybe_creator_anonymous_visitor(job.maybe_creator_anonymous_visitor_token.as_ref()) .maybe_creator_user(job.maybe_creator_user_token.as_ref()) - .maybe_engine_category(Some(MediaFileEngineCategory::Object)) .maybe_generation_provider(Some(GenerationProvider::Artcraft)) .maybe_prompt_token(job.maybe_prompt_token.as_ref()) .maybe_platform_type(job.maybe_platform_type) @@ -178,7 +184,17 @@ pub(crate) async fn upload_mesh_file( .media_file_origin_category(MediaFileOriginCategory::Inference) .media_file_type(media_file_type) .mime_type(mime_type) - .public_bucket_directory_hash(&public_upload_path) + .public_bucket_directory_hash(&public_upload_path); + + if is_gaussian_splat { + insert_builder = insert_builder + .media_file_origin_product_category(MediaFileOriginProductCategory::WorldGeneration); + } else { + insert_builder = insert_builder + .maybe_engine_category(Some(MediaFileEngineCategory::Object)); + } + + let media_token = insert_builder .insert_pool(&server_state.mysql_pool) .await .map_err(|err| { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/resolve_file_metadata.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/resolve_file_metadata.rs index a93e40b93e..df905251d2 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/resolve_file_metadata.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/resolve_file_metadata.rs @@ -86,6 +86,7 @@ pub fn resolve_file_metadata_with_file_name( let (mime_type, file_extension) = match extension.as_str() { "fbx" => ("model/fbx", FileExtension::Fbx), "obj" => ("model/obj", FileExtension::Obj), + "ply" => ("model/ply", FileExtension::Ply), "glb" => ("model/gltf-binary", FileExtension::Glb), _ => return None, };