diff --git a/Cargo.lock b/Cargo.lock index 854910e8bc..957d78280d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,6 +34,29 @@ dependencies = [ "smallvec", ] +[[package]] +name = "actix-files" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8c4f30e3272d7c345f88ae0aac3848507ef5ba871f9cc2a41c8085a0f0523b" +dependencies = [ + "actix-http", + "actix-service", + "actix-utils", + "actix-web", + "bitflags 2.11.0", + "bytes", + "derive_more 2.0.1", + "futures-core", + "http-range", + "log", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "v_htmlescape", +] + [[package]] name = "actix-http" version = "3.12.1" @@ -7673,7 +7696,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ - "proc-macro-crate 2.0.0", + "proc-macro-crate 3.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -11577,6 +11600,7 @@ checksum = "ef5430c8e36b713e13b48a9f709cc21e046723fe44ce34587b73a830203b533e" name = "storyteller-web" version = "0.0.1" dependencies = [ + "actix-files", "actix-http", "actix-multipart", "actix-service", @@ -13894,6 +13918,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "v_htmlescape" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e8257fbc510f0a46eb602c10215901938b5c2a7d5e70fc11483b1d3c9b5b18c" + [[package]] name = "value-bag" version = "1.12.0" diff --git a/Cargo.toml b/Cargo.toml index 251a0093cd..13f158e0a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -307,6 +307,7 @@ worldlabs_consumer_client = { path = "crates/api_clients/worldlabs_consumer_clie # Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace # If a crate needs to use an older/newer version, the dependency should be managed in that crate's own Cargo.toml actix-cors = "0.7.1" +actix-files = "0.6.9" actix-http = "3.12.1" actix-multipart = "0.7.2" actix-rt = "2.11.0" diff --git a/_docs/dev_setup_local_stack.md b/_docs/dev_setup_local_stack.md index c5765e0359..2f05f77668 100644 --- a/_docs/dev_setup_local_stack.md +++ b/_docs/dev_setup_local_stack.md @@ -121,12 +121,39 @@ live in the generated secrets file: `ACCESS_KEY`, `SECRET_KEY`, `REGION_NAME`, `STRIPE_ARTCRAFT_ACCOUNT_ID`, `STRIPE_ARTCRAFT_SECRET_KEY`, `STRIPE_ARTCRAFT_SECRET_WEBHOOK_KEY`. -**Placeholder values are safe**: none of these services is contacted at boot -(clients are constructed offline; Elasticsearch, S3/R2, Stripe, Resend, and -Google-cert fetches are all lazy). Generation stays safe by construction — -handlers only validate, bill the wallet, and insert job rows; providers are -contacted exclusively by separate worker binaries that local dev never runs, -so enqueued jobs just stay pending at zero cost. +**Placeholder values are safe at boot**: none of these services is contacted +at boot (clients are constructed offline; Elasticsearch, S3/R2, Stripe, +Resend, and Google-cert fetches are all lazy). But two earlier claims here +were wrong and are corrected: + +- **The omni_gen endpoints call providers synchronously in the HTTP handler** + (`/v1/omni_gen/generate/{image,video}` — the path the webapp uses). With + dummy keys the provider call fails and the submit returns HTTP 500 *after* + billing the wallet (the image pipeline has no refund; video refunds only + Kinovi). Only the legacy polling providers (GmiCloud, Grok, Kinovi orders, + WorldLabs) are worker-driven. `DEV_FAKE_GENERATION` (below) exists so local + submits never reach a real provider. +- **Uploads fail hard with dummy S3 creds**: every media upload endpoint + uploads to the bucket *before* inserting the DB row, and `rust-s3` is built + with `fail-on-err` — so the first upload returns HTTP 500. (Known seam; + uploads are not yet part of the local stack.) + +### Fully-local generation and media (`DEV_FAKE_GENERATION`) + +The generated secrets file enables three development-only settings (appended +automatically on bootstrap re-runs for older stacks): + +| Env var | Effect | +|-----------------------|----------------------------------------------------------------------------| +| `DEV_FAKE_GENERATION` | omni_gen image/video submits skip the external provider and insert a normal **pending** job with a `fake_…` external id. A resolver thread inside `storyteller-web` completes each one after `DEV_FAKE_GENERATION_RESOLVE_SECS` (default 6s, poll every `DEV_FAKE_GENERATION_POLL_SECS`, default 2s) with a placeholder result copied from `test_data/`. Prompts containing `simulate_artcraft_failure` (or `test_artcraft_failure`) fail instead, optionally with a `FrontendFailureCategory` word in the prompt (e.g. `simulate_artcraft_failure face_not_detected`). Hard-disabled outside `SERVER_ENVIRONMENT=Development` | +| `CDN_BASE_URL` | Overrides the compiled-in CDN hosts in `cdn_link.rs` for all media URLs in API responses. Must be scheme+host only (no path) — set to `http://localhost:12345` so media resolves against the backend itself | +| `LOCAL_MEDIA_ROOT` | Directory the backend serves at `/media` (dev-only actix-files mount), mirroring the public bucket layout: object path `/media/{…}` lives at `{LOCAL_MEDIA_ROOT}/media/{…}`. The bootstrap points it at `.devstack/media/` | + +With all three set, the full loop works offline: submit → pending job → +polling (`/v1/jobs/session`) → success with a `media_files` row whose URL the +backend itself serves — or a failed job with a real failure category. Credits +are genuinely deducted from the seeded wallet (no refund on fake results — +the demo wallet is large enough not to care). ### Auth on localhost @@ -170,6 +197,18 @@ so enqueued jobs just stay pending at zero cost. (`cargo run --bin dev-database-seed`) exists but hard-requires a `.env-secrets` file and also seeds model weights, so the HTTP path is preferred. +- Demo credits: signup creates **no wallet**, and the webapp gates the + generate button on wallet balance — so `seed_demo_user` also seeds an + `artcraft`-namespace wallet with `DEMO_CREDITS` (default 100,000) banked + credits, plus matching ledger rows. Idempotent: re-runs top the balance + back up to `DEMO_CREDITS` but never lower it. (Raw SQL, because the only + real credit-granting paths are Stripe webhooks and moderator tooling.) +- Demo gallery: `seed_demo_user` also seeds a handful of `media_files` image + rows (from `test_data/image/`) with their objects copied under + `.devstack/media/`, so the library isn't empty on a fresh stack. Rows mirror + what a real image upload writes (`media_class='image'` matters — the + library filters on it, and the column default `'unknown'` is invisible in + every tab). ## Frontend: `artcraft-webapp` @@ -200,9 +239,10 @@ change runtime behavior: | Seam | Today | |-----------------------------------------------|--------------------------------------------------------------| -| Media/CDN URLs in API responses | Compiled-in constants (`cdn_link.rs`) point dev builds at a public `pub-….r2.dev` bucket — media loads remotely even with a local backend | +| Media/CDN URLs in API responses | **Closed** when `CDN_BASE_URL` + `LOCAL_MEDIA_ROOT` are set (the bootstrap default): media URLs point at the backend's own `/media` mount. Unset, the compiled-in `cdn_link.rs` constants still point at the public `pub-….r2.dev` bucket | +| Media uploads | Upload endpoints push to R2/S3 *before* the DB insert and `rust-s3` is `fail-on-err` — with dummy creds every upload endpoint returns HTTP 500. A local S3 (`S3_COMPATIBLE_ENDPOINT_URL` → MinIO) or a filesystem storage mode is the missing piece | | Vite `/v1` proxy in webapp/website | Hardcoded `https://api.storyteller.ai` (bypassed in dev by `setDevelopment()`, but still the fallback path) | -| `SegmentationApi`, `GetCdnOrigin`, `API_TARGETS` | Hardcoded off-store production hosts | +| `SegmentationApi`, `GetCdnOrigin`, `API_TARGETS` | Hardcoded off-store production hosts (`GetCdnOrigin`/`Api.ts` still hardcode `cdn-2.fakeyou.com` for pagescene/moodboard surfaces, independent of the backend's `CDN_BASE_URL`) | | Elasticsearch-backed search | `search_featured` / `search_session` media files, `weights/search`, legacy `tts/search` fail without a local ES + `elasticsearch-cli` reindex (optional) | ## CI usage @@ -225,11 +265,16 @@ export MYSQL_ROOT_PASSWORD=root # GitHub Actions' preinstalled MySQL uses pass - `./script/bootstrap/dev_stack_doctor.sh` exits 0 with no FAILs. - `POST /v1/login` with the demo user returns `success: true` plus a `signed_session` (done automatically by `seed_demo_user.sh`). -- Webapp at `http://localhost:4201` can log in as `localdev1`. -- A generation enqueue returns `{"success": true, "inference_job_token": …}` - and no external provider is contacted (no workers running). -- Nothing hits `api.storyteller.ai` during the above (verify in the browser - network tab; media/CDN requests are the known exception — see seams table). +- Webapp at `http://localhost:4201` can log in as `localdev1`, has credits in + the top bar, and sees the seeded images in the library. +- A generation submit returns `{"success": true, "inference_job_token": …}`, + shows as pending for a few seconds, then completes with a placeholder + result (`DEV_FAKE_GENERATION`) — no external provider is contacted. A + prompt containing `simulate_artcraft_failure` fails with a real error + toast/category instead. +- Nothing hits `api.storyteller.ai` or the r2.dev CDN during the above + (verify in the browser network tab — media URLs should point at + `http://localhost:12345/media/…`). ## Corrections to older docs diff --git a/crates/schema/database/mysql_queries/src/queries/generic_inference/web/list_pending_fake_generation_jobs.rs b/crates/schema/database/mysql_queries/src/queries/generic_inference/web/list_pending_fake_generation_jobs.rs new file mode 100644 index 0000000000..8afe784fb4 --- /dev/null +++ b/crates/schema/database/mysql_queries/src/queries/generic_inference/web/list_pending_fake_generation_jobs.rs @@ -0,0 +1,64 @@ +use sqlx::{MySqlPool, Row}; + +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::prompts::PromptToken; +use tokens::tokens::users::UserToken; + +/// A pending job created by the DEV_FAKE_GENERATION short-circuit +/// (external_third_party_id starts with `fake_`), joined with its prompt text +/// so the dev resolver can honor the synthetic-failure prompt triggers. +pub struct PendingFakeGenerationJob { + pub job_token: InferenceJobToken, + /// e.g. "image_generation", "video_generation". + pub inference_category: String, + pub maybe_prompt_token: Option, + pub maybe_positive_prompt: Option, + pub maybe_creator_user_token: Option, +} + +/// List pending fake-generation jobs older than `min_age_seconds`, oldest +/// first. Development-only: production never writes `fake_` external ids. +/// +/// NB: runtime query (not the compile-time macro) because this dev-only SELECT +/// isn't in the sqlx offline cache. +pub async fn list_pending_fake_generation_jobs( + mysql_pool: &MySqlPool, + min_age_seconds: u32, + limit: u32, +) -> Result, sqlx::Error> { + let rows = sqlx::query( + r#" +SELECT + j.token AS job_token, + j.inference_category, + j.maybe_prompt_token, + j.maybe_creator_user_token, + p.maybe_positive_prompt +FROM generic_inference_jobs AS j +LEFT JOIN prompts AS p + ON p.token = j.maybe_prompt_token +WHERE j.status = 'pending' + AND j.maybe_external_third_party_id LIKE 'fake\_%' + AND j.created_at <= NOW() - INTERVAL ? SECOND +ORDER BY j.created_at ASC +LIMIT ? + "#) + .bind(min_age_seconds) + .bind(limit) + .fetch_all(mysql_pool) + .await?; + + let jobs = rows.into_iter() + .map(|row| PendingFakeGenerationJob { + job_token: InferenceJobToken::new(row.get::("job_token")), + inference_category: row.get::("inference_category"), + maybe_prompt_token: row.get::, _>("maybe_prompt_token") + .map(PromptToken::new), + maybe_positive_prompt: row.get::, _>("maybe_positive_prompt"), + maybe_creator_user_token: row.get::, _>("maybe_creator_user_token") + .map(UserToken::new), + }) + .collect(); + + Ok(jobs) +} diff --git a/crates/schema/database/mysql_queries/src/queries/generic_inference/web/mod.rs b/crates/schema/database/mysql_queries/src/queries/generic_inference/web/mod.rs index d998ba65db..39b38c8c35 100644 --- a/crates/schema/database/mysql_queries/src/queries/generic_inference/web/mod.rs +++ b/crates/schema/database/mysql_queries/src/queries/generic_inference/web/mod.rs @@ -8,6 +8,7 @@ pub mod job_status; pub mod kill_generic_inference_jobs; pub mod list_user_jobs_for_moderation; pub mod kill_jobs_in_development; +pub mod list_pending_fake_generation_jobs; pub mod list_session_jobs; pub mod mark_generic_inference_job_cancelled_by_user; pub mod mark_generic_inference_job_successfully_done_by_token; diff --git a/crates/service/web/storyteller_web/CLAUDE.md b/crates/service/web/storyteller_web/CLAUDE.md index 93bfbb00d5..00d6a27a2e 100644 --- a/crates/service/web/storyteller_web/CLAUDE.md +++ b/crates/service/web/storyteller_web/CLAUDE.md @@ -75,23 +75,23 @@ curl -s -c /tmp/cookies.txt -X POST http://localhost:12345/v1/login \ -H "Content-Type: application/json" \ -d '{"username_or_email":"newuser4","password":"newuser4"}' -# 2. Generate video (GmiCloud Seedance 2.0 Fast Global): +# 2. Generate video (Seedance 2.0 Fast): # Requires a UUID-format idempotency_token. curl -s -b /tmp/cookies.txt -X POST http://localhost:12345/v1/omni_gen/generate/video \ -H "Content-Type: application/json" \ -d '{ - "model": "seedance_2p0_fast_g", + "model": "seedance_2p0_fast", "prompt": "a corgi running through a field", "duration_seconds": 5, "aspect_ratio": "wide_sixteen_by_nine", "idempotency_token": "a1b2c3d4-e5f6-7890-abcd-ef1234567890" }' -# 3. Generate video (GmiCloud Seedance 2.0 Global, 720p): +# 3. Generate video (Seedance 2.0, 720p): curl -s -b /tmp/cookies.txt -X POST http://localhost:12345/v1/omni_gen/generate/video \ -H "Content-Type: application/json" \ -d '{ - "model": "seedance_2p0_g", + "model": "seedance_2p0", "prompt": "a corgi running through a field", "duration_seconds": 5, "resolution": "seven_twenty_p", diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 30fb9a4c93..17ad0f9e8e 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -167,6 +167,7 @@ tempfile.workspace = true # https://github.com/durch/rust-s3/pull/254 rust-s3.workspace = true +actix-files.workspace = true actix-http.workspace = true actix-multipart.workspace = true actix-service.workspace = true diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/media/cdn_link.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/media/cdn_link.rs index a51497caf7..cea2c027d8 100644 --- a/crates/service/web/storyteller_web/src/http_server/common_responses/media/cdn_link.rs +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/media/cdn_link.rs @@ -9,6 +9,34 @@ const STORYTELLER_CDN_STR: &str = "https://cdn-2.fakeyou.com"; const FAKEYOU_DEVELOPMENT_CDN_STR: &str = "https://pub-c8a4a5bdbdb048f286b77bdf9f786ff2.r2.dev"; const STORYTELLER_DEVELOPMENT_CDN_STR: &str = "https://pub-c8a4a5bdbdb048f286b77bdf9f786ff2.r2.dev"; +// Runtime override for the CDN base, taking precedence over every constant +// above for all domains and environments. The fully-local dev stack sets +// CDN_BASE_URL=http://localhost:12345 so media URLs resolve against the +// backend's own static /media mount (see add_service_routes) instead of the +// shared r2.dev bucket. Unset (production, and dev-against-remote) keeps the +// compiled-in hosts. +// +// The override must be scheme+host only: some consumers append the rooted +// object path with Url::set_path, which would silently drop a path component +// in the base, so a base with a path is rejected loudly instead. +// Leaked once so get_cdn_host can stay &'static str. +static CDN_BASE_URL_OVERRIDE_STR: Lazy> = Lazy::new(|| { + let base = std::env::var("CDN_BASE_URL") + .ok() + .map(|s| s.trim().trim_end_matches('/').to_string()) + .filter(|s| !s.is_empty())?; + let url = Url::parse(&base).expect("CDN_BASE_URL must be a valid absolute URL"); + if url.path() != "/" && !url.path().is_empty() { + panic!("CDN_BASE_URL must not include a path component (got '{}'): \ + media URL builders overwrite the path", url.path()); + } + Some(&*Box::leak(base.into_boxed_str())) +}); + +static CDN_BASE_URL_OVERRIDE: Lazy> = Lazy::new(|| { + CDN_BASE_URL_OVERRIDE_STR.map(|s| Url::parse(s).expect("validated above")) +}); + const FAKEYOU_CDN: Lazy = Lazy::new(|| Url::parse(FAKEYOU_CDN_STR) .expect("should never fail")); @@ -25,6 +53,9 @@ const STORYTELLER_DEVELOPMENT_CDN: Lazy = Lazy::new(|| Url::parse(STORYTELL // TODO(bt,2025-01-31): Perhaps this should be config driven and configurable at runtime instead of hardcoded. pub fn get_cdn_host(media_domain: MediaDomain, server_environment: ServerEnvironment) -> &'static str { + if let Some(base) = *CDN_BASE_URL_OVERRIDE_STR { + return base; + } match (media_domain, server_environment) { (MediaDomain::FakeYou, ServerEnvironment::Development) => FAKEYOU_DEVELOPMENT_CDN_STR, (MediaDomain::FakeYou, ServerEnvironment::Production) => FAKEYOU_CDN_STR, @@ -34,6 +65,9 @@ pub fn get_cdn_host(media_domain: MediaDomain, server_environment: ServerEnviron } pub fn new_cdn_url(media_domain: MediaDomain, server_environment: ServerEnvironment) -> Url { + if let Some(url) = CDN_BASE_URL_OVERRIDE.as_ref() { + return url.clone(); + } match (media_domain, server_environment) { (MediaDomain::FakeYou, ServerEnvironment::Development) => FAKEYOU_DEVELOPMENT_CDN.clone(), (MediaDomain::FakeYou, ServerEnvironment::Production) => FAKEYOU_CDN.clone(), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/image/pipeline_v2/run_pipeline_v2.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/image/pipeline_v2/run_pipeline_v2.rs index 8f856b724c..dd9b8366b8 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/image/pipeline_v2/run_pipeline_v2.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/image/pipeline_v2/run_pipeline_v2.rs @@ -7,13 +7,14 @@ use artcraft_router::client::router_client::RouterClient; use artcraft_router::client::router_fal_client::RouterFalClient; use artcraft_router::client::router_seedance2pro_client::RouterSeedance2ProClient; use artcraft_router::generate::generate_image::generate_image_request_builder::GenerateImageRequestBuilder; -use artcraft_router::generate::generate_image::generate_image_response::GenerateImageResponse; +use artcraft_router::generate::generate_image::generate_image_response::{FalImageResponsePayload, GenerateImageResponse}; use artcraft_router::generate::generate_image::image_generation_draft_context::ImageGenerationDraftContext; use artcraft_router::generate::generate_image::image_generation_draft_or_request::ImageGenerationDraftOrRequest; use artcraft_router::generate::generate_image::image_generation_request::ImageGenerationRequest; use seedance2pro_client::creds::seedance2pro_session::Seedance2ProSession; use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::users::UserToken; +use uuid_utils::uuid::generate_random_uuid; use crate::billing::wallets::attempt_wallet_deduction::attempt_wallet_deduction_else_common_web_error; use crate::http_server::common_responses::common_web_error::CommonWebError; @@ -173,6 +174,21 @@ async fn finalize_and_generate( server_state: &ServerState, resolved_media: &MediaFilesAsCdnUrlListAndMap, ) -> Result { + // Dev-only: skip the external provider entirely and hand back a synthetic + // Fal enqueue. The handler then inserts a normal pending job (external id + // `fake_…`), which the dev fake-job resolver thread later completes. + if server_state.dev_fake_generation { + let fake_request_id = format!("fake_{}", generate_random_uuid()); + info!("DEV_FAKE_GENERATION: skipping image provider call, request_id={}", fake_request_id); + return Ok(GenerateImageResponse::Fal(FalImageResponsePayload { + request_id: Some(fake_request_id), + gateway_request_id: None, + maybe_status_url: None, + maybe_response_url: None, + maybe_outbound_request: None, + })); + } + let provider = draft_or_request.get_provider(); let client = build_router_client(provider, server_state)?; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/pipeline_v2/run_pipeline_v2.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/pipeline_v2/run_pipeline_v2.rs index 04893ed21f..a7cac68d0b 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/pipeline_v2/run_pipeline_v2.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/pipeline_v2/run_pipeline_v2.rs @@ -6,12 +6,13 @@ use log::{error, info, warn}; use artcraft_router::api::router_video_model::RouterVideoModel; use artcraft_router::api::router_provider::RouterProvider; use artcraft_router::generate::generate_video::generate_video_request_builder::GenerateVideoRequestBuilder; -use artcraft_router::generate::generate_video::generate_video_response::GenerateVideoResponse; +use artcraft_router::generate::generate_video::generate_video_response::{FalVideoResponsePayload, GenerateVideoResponse}; use artcraft_router::generate::generate_video::video_generation_draft_context::VideoGenerationDraftContext; use artcraft_router::generate::generate_video::video_generation_draft_or_request::VideoGenerationDraftOrRequest; use tokens::tokens::characters::CharacterToken; use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::users::UserToken; +use uuid_utils::uuid::generate_random_uuid; use crate::http_server::common_responses::common_web_error::CommonWebError; use crate::http_server::endpoint_helpers::refund_wallet_after_api_failure::refund_wallet_after_api_failure; @@ -220,6 +221,22 @@ async fn upload_and_generate( kinovi_account: KinoviAccount, ) -> Result { + // Dev-only: skip the external provider entirely (including draft media + // uploads) and hand back a synthetic Fal enqueue. The handler then inserts + // a normal pending job (external id `fake_…`), which the dev fake-job + // resolver thread later completes. + if server_state.dev_fake_generation { + let fake_request_id = format!("fake_{}", generate_random_uuid()); + info!("DEV_FAKE_GENERATION: skipping video provider call, request_id={}", fake_request_id); + return Ok(GenerateVideoResponse::Fal(FalVideoResponsePayload { + request_id: Some(fake_request_id), + gateway_request_id: None, + maybe_status_url: None, + maybe_response_url: None, + maybe_outbound_request: None, + })); + } + let provider = draft_or_request.get_provider(); let client = build_router_client(provider, server_state, kinovi_account)?; diff --git a/crates/service/web/storyteller_web/src/http_server/routes/add_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/add_routes.rs index e739fd4b65..b09710c9e7 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/add_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/add_routes.rs @@ -4,6 +4,7 @@ use actix_web::dev::{ServiceRequest, ServiceResponse}; use actix_web::error::Error; use actix_web::App; +use log::{info, warn}; use server_environment::ServerEnvironment; use crate::http_server::routes::application_routes::add_application_routes::add_application_routes; @@ -22,13 +23,45 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> >, { let mut app = app; - + app = add_legacy_routes(app); // various legacy routes, mostly deprecated app = add_application_routes(app); // Primary product service area routes + app = add_local_media_route(app, server_environment); // Dev-only static media (fully-local stack) app = add_service_routes(app); // Essential service routes (status, health, info, etc.) - + app } - - +/// Dev-only: serve media files from a local directory at /media, mirroring the +/// public bucket's layout, so a fully-local stack (CDN_BASE_URL pointed at this +/// server) can render seeded and fake-generated media without any cloud bucket. +/// LOCAL_MEDIA_ROOT is the bucket root: an object with rooted path +/// /media/{...} is read from {LOCAL_MEDIA_ROOT}/media/{...}. +/// No-op in production and when LOCAL_MEDIA_ROOT is unset. +fn add_local_media_route (app: App, server_environment: ServerEnvironment) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + if !server_environment.is_development() { + return app; + } + let root = match std::env::var("LOCAL_MEDIA_ROOT") { + Ok(root) if !root.trim().is_empty() => root, + _ => return app, + }; + let media_dir = std::path::Path::new(&root).join("media"); + if let Err(err) = std::fs::create_dir_all(&media_dir) { + // Fall back to remote CDN behavior rather than refusing to boot. + warn!("LOCAL_MEDIA_ROOT set but '{}' is not creatable ({}); /media not mounted", media_dir.display(), err); + return app; + } + info!("Dev local media: serving {} at /media", media_dir.display()); + app.service(actix_files::Files::new("/media", media_dir)) +} diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index 6bb5bbd3b1..597167a1b5 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -55,6 +55,7 @@ use crate::startup::setup_disabled_endpoints::read_disabled_endpoints; use crate::startup::setup_metrics::build_metrics; use crate::state::server_state::ServerState; use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; +use crate::threads::dev_fake_generation_resolver_thread::dev_fake_generation_resolver_thread; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::threads::poll_model_token_info_thread::poll_model_token_info_thread; @@ -148,6 +149,16 @@ async fn main() -> AnyhowResult<()> { poll_model_token_info_thread(model_token_info_cache_clone, mysql_pool_clone).await; }); + if server_state.dev_fake_generation { + info!("Spawning dev fake-generation resolver thread (DEV_FAKE_GENERATION=true)."); + + let mysql_pool_clone = server_state.mysql_pool.clone(); + + tokio_runtime.spawn(async { + dev_fake_generation_resolver_thread(mysql_pool_clone).await; + }); + } + // ==================== Metrics worker ==================== // let (metrics_collector, maybe_metrics_worker) = diff --git a/crates/service/web/storyteller_web/src/startup/build_dependencies.rs b/crates/service/web/storyteller_web/src/startup/build_dependencies.rs index 46229eb3bf..b2fdd499f7 100644 --- a/crates/service/web/storyteller_web/src/startup/build_dependencies.rs +++ b/crates/service/web/storyteller_web/src/startup/build_dependencies.rs @@ -9,7 +9,7 @@ use cloud_storage::legacy_bucket_client::LegacyBucketClient; use elasticsearch::http::transport::Transport; use elasticsearch::Elasticsearch; use errors::AnyhowResult; -use log::info; +use log::{info, warn}; use memory_caching::arc_ttl_sieve::ArcTtlSieve; use memory_caching::single_item_ttl_cache::SingleItemTtlCache; use mysql_queries::mediators::badge_granter::BadgeGranter; @@ -192,6 +192,15 @@ pub async fn setup_dependencies(server_hostname: &str) -> AnyhowResult AnyhowResult pending -> resolve flow. + pub dev_fake_generation: bool, + /// Feature flags will allow us to restart the service with different conditions embedded in the code. pub flags: StaticFeatureFlags, diff --git a/crates/service/web/storyteller_web/src/threads/dev_fake_generation_resolver_thread.rs b/crates/service/web/storyteller_web/src/threads/dev_fake_generation_resolver_thread.rs new file mode 100644 index 0000000000..27bb902527 --- /dev/null +++ b/crates/service/web/storyteller_web/src/threads/dev_fake_generation_resolver_thread.rs @@ -0,0 +1,202 @@ +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use log::{error, info}; +use sqlx::MySqlPool; + +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use bucket_paths::path_conventions::video_thumbnail_suffixes::{VIDEO_ANIMATED_GIF_THUMBNAIL_SUFFIX, VIDEO_STATIC_JPG_THUMBNAIL_SUFFIX}; +use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::generation_provider::GenerationProvider; +use errors::AnyhowResult; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use mysql_queries::queries::generic_inference::job::mark_job_failed_by_token::{mark_job_failed_by_token, MarkJobFailedByTokenArgs}; +use mysql_queries::queries::generic_inference::web::list_pending_fake_generation_jobs::{list_pending_fake_generation_jobs, PendingFakeGenerationJob}; +use mysql_queries::queries::generic_inference::web::mark_generic_inference_job_successfully_done_by_token_with_executor::{ + mark_generic_inference_job_successfully_done_by_token_with_executor, + MarkGenericInferenceJobSuccessfullyDoneByTokenWithExecutorArgs, +}; +use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder; +use tokens::tokens::media_files::MediaFileToken; + +use crate::http_server::endpoints::generate::common::job_failure_test::test_synthetic_failure_reason; + +/// Placeholder result assets, relative to the repo root (the server's working +/// directory — its config search path already depends on that). +const FAKE_IMAGE_RESULT_SOURCE: &str = "test_data/image/juno.jpg"; +const FAKE_VIDEO_RESULT_SOURCE: &str = "test_data/video/mp4/golden_sun_garoh.mp4"; + +const JOBS_PER_TICK: u32 = 25; + +/// Dev-only companion to DEV_FAKE_GENERATION (see ServerState): polls for +/// pending jobs whose external id is `fake_…` and, once they are older than +/// DEV_FAKE_GENERATION_RESOLVE_SECS, resolves them the way a real provider +/// callback would — a `media_files` row backed by a placeholder asset under +/// LOCAL_MEDIA_ROOT for success, or a synthetic failure when the prompt +/// contains the `simulate_artcraft_failure` / `test_artcraft_failure` +/// triggers (with an optional FrontendFailureCategory word, same as the +/// legacy handlers). +pub async fn dev_fake_generation_resolver_thread(mysql_pool: MySqlPool) { + let resolve_after = easyenv::get_env_duration_seconds_or_default( + "DEV_FAKE_GENERATION_RESOLVE_SECS", Duration::from_secs(6)); + + let poll_interval = easyenv::get_env_duration_seconds_or_default( + "DEV_FAKE_GENERATION_POLL_SECS", Duration::from_secs(2)); + + info!("Dev fake-generation resolver running: jobs resolve after ~{}s (poll every {}s).", + resolve_after.as_secs(), poll_interval.as_secs()); + + loop { + tokio::time::sleep(poll_interval).await; + + let jobs = match list_pending_fake_generation_jobs( + &mysql_pool, + resolve_after.as_secs() as u32, + JOBS_PER_TICK, + ).await { + Ok(jobs) => jobs, + Err(err) => { + error!("Dev fake-generation resolver failed to list pending jobs: {:?}", err); + continue; + } + }; + + for job in jobs { + let job_token = job.job_token.clone(); + if let Err(err) = resolve_job(&mysql_pool, &job, resolve_after).await { + error!("Dev fake-generation resolver failed on job {}: {:?}", job_token.as_str(), err); + // Fail the job so the frontend sees a terminal state instead of an + // eternally-pending spinner (and so we don't retry it every tick). + let mark_result = mark_job_failed_by_token(MarkJobFailedByTokenArgs { + pool: &mysql_pool, + job_token: &job_token, + maybe_public_failure_reason: Some("Dev fake generation could not produce a result."), + internal_debugging_failure_reason: &format!("dev fake resolver error: {:?}", err), + maybe_frontend_failure_category: None, + }).await; + if let Err(mark_err) = mark_result { + error!("Also failed to mark job {} failed: {:?}", job_token.as_str(), mark_err); + } + } + } + } +} + +async fn resolve_job( + mysql_pool: &MySqlPool, + job: &PendingFakeGenerationJob, + simulated_duration: Duration, +) -> AnyhowResult<()> { + let prompt = job.maybe_positive_prompt.as_deref().unwrap_or(""); + + if let Some(failure) = test_synthetic_failure_reason(prompt) { + mark_job_failed_by_token(MarkJobFailedByTokenArgs { + pool: mysql_pool, + job_token: &job.job_token, + maybe_public_failure_reason: failure.frontend_failure_message.as_deref(), + internal_debugging_failure_reason: "DEV_FAKE_GENERATION synthetic failure (prompt trigger)", + maybe_frontend_failure_category: Some(failure.frontend_failure_category), + }).await?; + info!("Dev fake-generation: job {} failed synthetically ({:?}).", + job.job_token.as_str(), failure.frontend_failure_category); + return Ok(()); + } + + let media_token = insert_fake_result_media(mysql_pool, job).await?; + + mark_generic_inference_job_successfully_done_by_token_with_executor( + MarkGenericInferenceJobSuccessfullyDoneByTokenWithExecutorArgs { + executor: mysql_pool, + token: &job.job_token, + maybe_entity_type: Some(InferenceResultType::MediaFile), + maybe_entity_token: Some(media_token.as_str()), + total_job_duration: Some(simulated_duration), + inference_duration: Some(simulated_duration), + }, + ).await?; + + info!("Dev fake-generation: job {} completed with media file {}.", + job.job_token.as_str(), media_token.as_str()); + Ok(()) +} + +/// Copy a placeholder asset to a fresh bucket path under LOCAL_MEDIA_ROOT and +/// insert the `media_files` row, mirroring what the Fal webhook success path +/// does with a real provider result (minus the R2 upload). +async fn insert_fake_result_media( + mysql_pool: &MySqlPool, + job: &PendingFakeGenerationJob, +) -> AnyhowResult { + let is_video = job.inference_category.contains("video"); + + let (source_path, extension, media_class, media_type, mime_type, product_category) = if is_video { + (FAKE_VIDEO_RESULT_SOURCE, ".mp4", MediaFileClass::Video, MediaFileType::Mp4, + "video/mp4", MediaFileOriginProductCategory::VideoGeneration) + } else { + (FAKE_IMAGE_RESULT_SOURCE, ".jpg", MediaFileClass::Image, MediaFileType::Jpg, + "image/jpeg", MediaFileOriginProductCategory::ImageGeneration) + }; + + let media_root = std::env::var("LOCAL_MEDIA_ROOT") + .ok() + .filter(|root| !root.trim().is_empty()) + .ok_or_else(|| anyhow::anyhow!( + "LOCAL_MEDIA_ROOT must be set for DEV_FAKE_GENERATION results to be viewable"))?; + + let file_bytes = tokio::fs::read(source_path).await + .map_err(|err| anyhow::anyhow!( + "cannot read placeholder asset {} (server must run from the repo root): {}", source_path, err))?; + + let bucket_path = MediaFileBucketPath::generate_new(Some("fake_"), Some(extension)); + let disk_path = local_disk_path(&media_root, &bucket_path.get_full_object_path_str()); + + if let Some(parent) = disk_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::write(&disk_path, &file_bytes).await?; + + if is_video { + // The video thumbnail job never runs locally, so drop the thumbnail + // siblings the gallery expects next to the object. The animated "gif" is + // really JPEG bytes — browsers decode content by magic bytes, and a + // still preview is plenty for dev. + let thumb_bytes = tokio::fs::read(FAKE_IMAGE_RESULT_SOURCE).await?; + let base = disk_path.to_string_lossy().to_string(); + tokio::fs::write(format!("{base}{VIDEO_STATIC_JPG_THUMBNAIL_SUFFIX}"), &thumb_bytes).await?; + tokio::fs::write(format!("{base}{VIDEO_ANIMATED_GIF_THUMBNAIL_SUFFIX}"), &thumb_bytes).await?; + } + + let checksum = sha256_hash_bytes(&file_bytes)?; + + let media_token = MediaFileInsertBuilder::new() + .checksum_sha2(&checksum) + .creator_ip_address("127.0.0.1") + .file_size_bytes(file_bytes.len() as u64) + .maybe_creator_user(job.maybe_creator_user_token.as_ref()) + .maybe_generation_provider(Some(GenerationProvider::Artcraft)) + .maybe_prompt_token(job.maybe_prompt_token.as_ref()) + .media_file_class(media_class) + .media_file_origin_category(MediaFileOriginCategory::Inference) + .media_file_origin_product_category(product_category) + .media_file_type(media_type) + .mime_type(mime_type) + .public_bucket_directory_hash(&bucket_path) + .insert_pool(mysql_pool) + .await + .map_err(|err| anyhow::anyhow!("failed to insert fake result media file: {:?}", err))?; + + Ok(media_token) +} + +/// {LOCAL_MEDIA_ROOT} + rooted object path (e.g. "/media/a/b/…/fake_….jpg"). +fn local_disk_path(media_root: &str, rooted_object_path: &str) -> PathBuf { + let mut path = Path::new(media_root).to_path_buf(); + for segment in rooted_object_path.split('/').filter(|s| !s.is_empty()) { + path.push(segment); + } + path +} diff --git a/crates/service/web/storyteller_web/src/threads/mod.rs b/crates/service/web/storyteller_web/src/threads/mod.rs index 5804badbad..2d1e60c47b 100644 --- a/crates/service/web/storyteller_web/src/threads/mod.rs +++ b/crates/service/web/storyteller_web/src/threads/mod.rs @@ -1,3 +1,4 @@ pub mod db_health_checker_thread; +pub mod dev_fake_generation_resolver_thread; pub mod poll_ip_banlist_thread; -pub mod poll_model_token_info_thread; \ No newline at end of file +pub mod poll_model_token_info_thread; diff --git a/script/bootstrap/bootstrap_dev_stack.sh b/script/bootstrap/bootstrap_dev_stack.sh index 5909792275..acae3c5aaf 100755 --- a/script/bootstrap/bootstrap_dev_stack.sh +++ b/script/bootstrap/bootstrap_dev_stack.sh @@ -249,6 +249,7 @@ write_secrets_env_if_missing() { step "Development secrets file" if [ -f "${SECRETS_ENV_FILE}" ]; then log "Already exists: ${SECRETS_ENV_FILE} (leaving it untouched)." + add_dev_stack_keys_to_existing_secrets_env return fi @@ -256,9 +257,7 @@ write_secrets_env_if_missing() { # (build_dependencies.rs / setup_inference_providers.rs / setup_stripe_*.rs) # but is NOT supplied by the checked-in config files. None of them is # contacted at boot — placeholder values are enough to run the server; the - # endpoints that would use them fail lazily if exercised. Generation is still - # safe: providers are only contacted by the separate job worker binaries, - # which local dev does not run. + # endpoints that would use them fail lazily if exercised. # # NOTE: config loading is FIRST-VALUE-WINS (dotenv never overrides an # already-set key), so values here cannot override anything already set in @@ -281,8 +280,11 @@ W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dummy-local-dev-public # Email (Resend) RESEND_API_KEY=dummy-local-dev -# Generation providers (only job workers ever contact these; workers are not -# run in local dev, so enqueued jobs simply stay pending) +# Generation providers. NB: the omni_gen endpoints (webapp image/video +# generation) contact providers SYNCHRONOUSLY in the request handler, so with +# dummy keys a real submit would 500 — DEV_FAKE_GENERATION below avoids that. +# The legacy polling providers are only contacted by worker binaries, which +# local dev does not run. FAL_API_KEY=dummy-local-dev GMICLOUD_API_KEY=dummy-local-dev GROK_API_KEY=dummy-local-dev @@ -300,9 +302,40 @@ STRIPE_ARTCRAFT_ACCOUNT_ID=acct_dummylocaldev STRIPE_ARTCRAFT_SECRET_KEY=sk_test_dummylocaldev STRIPE_ARTCRAFT_SECRET_WEBHOOK_KEY=whsec_dummylocaldev EOF + dev_stack_keys_block >> "${SECRETS_ENV_FILE}" log "Wrote ${SECRETS_ENV_FILE}" } +# The fully-local media + fake-generation settings (development-only; the +# server hard-ignores DEV_FAKE_GENERATION outside SERVER_ENVIRONMENT=Development). +dev_stack_keys_block() { + cat <&2; } die() { echo "[bootstrap] ERROR: $*" >&2; exit 1; } @@ -55,6 +66,21 @@ confirm() { [[ "${reply}" =~ ^[Yy] ]] } +# Generate a token in the backend's format: {prefix}{crockford-lower entropy} +# padded to the same TOTAL length the Rust generators use (tokens crate, +# impl_crockford_generator!). Alphabet excludes i/l/o/u like Crockford base32. +new_dev_token() { + local prefix="$1" total_length="$2" + local charset="0123456789abcdefghjkmnpqrstvwxyz" + local count=$((total_length - ${#prefix})) + local entropy="" + local i + for ((i = 0; i < count; i++)); do + entropy+="${charset:RANDOM%32:1}" + done + echo "${prefix}${entropy}" +} + # Run a statement (or stdin) as the storyteller app user against the dev DB. # MYSQL_PWD keeps the password off the process list. mysql_app() { diff --git a/script/bootstrap/dev_stack_doctor.sh b/script/bootstrap/dev_stack_doctor.sh index 525e3250e4..9f42234a55 100755 --- a/script/bootstrap/dev_stack_doctor.sh +++ b/script/bootstrap/dev_stack_doctor.sh @@ -79,6 +79,20 @@ if mysql_app_reachable; then demo_count="$(mysql_app -N -e "SELECT COUNT(*) FROM users WHERE username='${DEMO_USERNAME}'" 2>/dev/null || echo 0)" if [ "${demo_count}" -gt 0 ]; then pass "demo user: '${DEMO_USERNAME}' exists" + + credits="$(mysql_app -N -e "SELECT COALESCE(MAX(banked_credits + monthly_credits), 0) FROM wallets WHERE wallet_namespace='artcraft' AND owner_user_token=(SELECT token FROM users WHERE username='${DEMO_USERNAME}')" 2>/dev/null || echo 0)" + if [ "${credits}" -gt 0 ]; then + pass "demo wallet: '${DEMO_USERNAME}' has ${credits} credits" + else + warn_check "demo wallet: '${DEMO_USERNAME}' has no credits — generation flows are blocked (re-run seed_demo_user.sh)" + fi + + media_count="$(mysql_app -N -e "SELECT COUNT(*) FROM media_files WHERE maybe_creator_user_token=(SELECT token FROM users WHERE username='${DEMO_USERNAME}')" 2>/dev/null || echo 0)" + if [ "${media_count}" -gt 0 ]; then + pass "demo media: '${DEMO_USERNAME}' has ${media_count} media rows" + else + warn_check "demo media: gallery is empty (re-run seed_demo_user.sh)" + fi else warn_check "demo user: '${DEMO_USERNAME}' not created yet (./script/bootstrap/seed_demo_user.sh, backend must be running)" fi @@ -97,6 +111,12 @@ echo "--- Backend ---" if [ -f "${SECRETS_ENV_FILE}" ]; then pass "secrets env: ${SECRETS_ENV_FILE#"${root_dir}"/} exists" + + if grep -q '^[[:space:]]*LOCAL_MEDIA_ROOT=' "${SECRETS_ENV_FILE}"; then + pass "secrets env: fully-local dev stack keys present (DEV_FAKE_GENERATION, CDN_BASE_URL, LOCAL_MEDIA_ROOT)" + else + warn_check "secrets env: fully-local dev stack keys missing — generation/gallery use remote CDN and real providers (re-run bootstrap to append)" + fi else fail "secrets env: missing — the server aborts at boot on ~19 required vars (re-run bootstrap)" fi diff --git a/script/bootstrap/seed_demo_user.sh b/script/bootstrap/seed_demo_user.sh index 6fb396f1be..2944188e5f 100755 --- a/script/bootstrap/seed_demo_user.sh +++ b/script/bootstrap/seed_demo_user.sh @@ -50,7 +50,7 @@ create_response="$(curl -sS -X POST "${DEV_BACKEND_URL}/v1/create_account" \ if [ "$(echo "${create_response}" | jq -r '.success // false')" = "true" ]; then log "Account created." else - error_code="$(echo "${create_response}" | jq -r '.error_code // .error_code_str // empty')" + error_code="$(echo "${create_response}" | jq -r '.error_type // .error_code // .error_code_str // empty')" case "${error_code}" in *[Uu]sername[Tt]aken*|*[Ee]mail[Tt]aken*) log "Account already exists (${error_code}) — continuing." @@ -76,6 +76,94 @@ if [ -z "$(echo "${login_response}" | jq -r '.signed_session // empty')" ]; then die "Login succeeded but returned no signed_session: ${login_response}" fi +# --- Credits wallet --------------------------------------------------------- +# Generation submits deduct from the user's artcraft wallet and the webapp +# gates the generate button on the balance, so a user without credits cannot +# exercise any generation flow. Signup does not create a wallet; seed one. +# Raw SQL (not an endpoint): the only credit-granting paths are Stripe +# webhooks and moderator tooling, neither of which works locally. +log "Seeding credits wallet (${DEMO_CREDITS} banked credits)..." + +user_token="$(mysql_app -N -e "SELECT token FROM users WHERE username = '${DEMO_USERNAME}' LIMIT 1")" +[ -n "${user_token}" ] || die "Could not find users row for '${DEMO_USERNAME}'." + +wallet_row="$(mysql_app -N -e "SELECT token, banked_credits FROM wallets WHERE wallet_namespace = 'artcraft' AND owner_user_token = '${user_token}' LIMIT 1")" + +if [ -z "${wallet_row}" ]; then + wallet_token="$(new_dev_token "wallet_" 32)" + create_ledger_token="$(new_dev_token "wle_" 32)" + credit_ledger_token="$(new_dev_token "wle_" 32)" + mysql_app -e " + INSERT INTO wallets SET token='${wallet_token}', wallet_namespace='artcraft', owner_user_token='${user_token}', banked_credits=${DEMO_CREDITS}, monthly_credits=0; + INSERT INTO wallet_ledger_entries SET token='${create_ledger_token}', wallet_token='${wallet_token}', entry_type='create', credits_delta=0, banked_credits_before=0, banked_credits_after=0, monthly_credits_before=0, monthly_credits_after=0; + INSERT INTO wallet_ledger_entries SET token='${credit_ledger_token}', wallet_token='${wallet_token}', entry_type='credit_banked', maybe_entity_ref='dev_bootstrap_seed', credits_delta=${DEMO_CREDITS}, banked_credits_before=0, banked_credits_after=${DEMO_CREDITS}, monthly_credits_before=0, monthly_credits_after=0; + " || die "Failed to insert the demo wallet." + log "Wallet created with ${DEMO_CREDITS} banked credits." +else + wallet_token="$(echo "${wallet_row}" | cut -f1)" + balance="$(echo "${wallet_row}" | cut -f2)" + if [ "${balance}" -lt "${DEMO_CREDITS}" ]; then + credit_ledger_token="$(new_dev_token "wle_" 32)" + delta=$((DEMO_CREDITS - balance)) + mysql_app -e " + UPDATE wallets SET banked_credits=${DEMO_CREDITS}, version=version+1 WHERE token='${wallet_token}' LIMIT 1; + INSERT INTO wallet_ledger_entries SET token='${credit_ledger_token}', wallet_token='${wallet_token}', entry_type='credit_banked', maybe_entity_ref='dev_bootstrap_topup', credits_delta=${delta}, banked_credits_before=${balance}, banked_credits_after=${DEMO_CREDITS}, monthly_credits_before=0, monthly_credits_after=0; + " || die "Failed to top up the demo wallet." + log "Wallet topped up: ${balance} -> ${DEMO_CREDITS} banked credits." + else + log "Wallet already has ${balance} banked credits — leaving as-is." + fi +fi + +# --- Gallery media ---------------------------------------------------------- +# Seed a few images so the library/gallery isn't empty on a fresh stack. The +# files live under .devstack/media (served by the backend at /media when +# LOCAL_MEDIA_ROOT + CDN_BASE_URL are set — bootstrap writes both into the +# generated secrets env). Rows mirror what a real image upload would write. +log "Seeding demo gallery media..." + +seed_sources=( + "test_data/image/juno.jpg|Dev seed - Juno 1" + "test_data/image/mochi.jpg|Dev seed - Mochi 1" + "test_data/image/juno.jpg|Dev seed - Juno 2" + "test_data/image/mochi.jpg|Dev seed - Mochi 2" + "test_data/image/juno.jpg|Dev seed - Juno 3" + "test_data/image/mochi.jpg|Dev seed - Mochi 3" +) + +existing_media="$(mysql_app -N -e "SELECT COUNT(*) FROM media_files WHERE maybe_creator_user_token='${user_token}' AND maybe_title LIKE 'Dev seed %'" 2>/dev/null || echo 0)" +if [ "${existing_media}" -ge "${#seed_sources[@]}" ]; then + log "Gallery already has ${existing_media} seeded media rows — leaving as-is." +else + for entry in "${seed_sources[@]}"; do + source_rel="${entry%%|*}" + title="${entry#*|}" + source_path="${root_dir}/${source_rel}" + [ -f "${source_path}" ] || die "Seed image missing: ${source_path}" + + # Mirror MediaFileBucketPath: /media/{h0..h4}/{hash}/image_{hash}.jpg + hash="$(new_dev_token "" 32)" + fanout="${hash:0:1}/${hash:1:1}/${hash:2:1}/${hash:3:1}/${hash:4:1}" + object_dir="${DEV_MEDIA_ROOT}/media/${fanout}/${hash}" + mkdir -p "${object_dir}" + cp "${source_path}" "${object_dir}/image_${hash}.jpg" + + sha="$(sha256sum "${source_path}" | cut -d' ' -f1)" + size="$(stat -c%s "${source_path}" 2>/dev/null || stat -f%z "${source_path}")" + media_token="$(new_dev_token "m_" 32)" + + mysql_app -e " + INSERT INTO media_files SET token='${media_token}', maybe_creator_user_token='${user_token}', + origin_category='upload', origin_product_category='unknown', media_class='image', media_type='jpg', + maybe_mime_type='image/jpeg', file_size_bytes=${size}, checksum_sha2='${sha}', + public_bucket_directory_hash='${hash}', maybe_public_bucket_prefix='image_', maybe_public_bucket_extension='.jpg', + creator_ip_address='127.0.0.1', creator_set_visibility='public', is_user_upload=1, is_intermediate_system_file=0, + maybe_title='${title}'; + " || die "Failed to insert seeded media row for ${title}." + done + log "Seeded ${#seed_sources[@]} gallery images under ${DEV_MEDIA_ROOT}." +fi + log "Demo user is ready." log " Username: ${DEMO_USERNAME}" log " Password: ${DEMO_PASSWORD}" diff --git a/script/bootstrap/windows/bootstrap_dev_stack.ps1 b/script/bootstrap/windows/bootstrap_dev_stack.ps1 index fc518414f9..76e8d2c3ea 100644 --- a/script/bootstrap/windows/bootstrap_dev_stack.ps1 +++ b/script/bootstrap/windows/bootstrap_dev_stack.ps1 @@ -323,13 +323,13 @@ function Seed-RolesAndBadges { function Write-SecretsEnvIfMissing { if (Test-Path $SecretsEnvFile) { Write-Log "Already exists: $SecretsEnvFile (leaving it untouched)." + Add-DevStackKeysToExistingSecretsEnv return } # Every var below is read with get_env_string_required at server startup but # is NOT supplied by the checked-in config files. None is contacted at boot - - # placeholders are enough to run the server. Providers are only contacted by - # job worker binaries, which local dev does not run. + # placeholders are enough to run the server. # # NOTE: config loading is FIRST-VALUE-WINS (dotenv never overrides a set # key), so values here cannot override storyteller-web.development.env. @@ -358,8 +358,11 @@ W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dummy-local-dev-public # Email (Resend) RESEND_API_KEY=dummy-local-dev -# Generation providers (only job workers ever contact these; workers are not -# run in local dev, so enqueued jobs simply stay pending) +# Generation providers. NB: the omni_gen endpoints (webapp image/video +# generation) contact providers SYNCHRONOUSLY in the request handler, so with +# dummy keys a real submit would 500 - DEV_FAKE_GENERATION below avoids that. +# The legacy polling providers are only contacted by worker binaries, which +# local dev does not run. FAL_API_KEY=dummy-local-dev GMICLOUD_API_KEY=dummy-local-dev GROK_API_KEY=dummy-local-dev @@ -376,11 +379,42 @@ STRIPE_FAKEYOU_ACCOUNT_ID=acct_dummylocaldev STRIPE_ARTCRAFT_ACCOUNT_ID=acct_dummylocaldev STRIPE_ARTCRAFT_SECRET_KEY=sk_test_dummylocaldev STRIPE_ARTCRAFT_SECRET_WEBHOOK_KEY=whsec_dummylocaldev + +$(Get-DevStackKeysBlock) "@ Set-Content -Path $SecretsEnvFile -Value $content -Encoding ascii Write-Log "Wrote $SecretsEnvFile" } +# The fully-local media + fake-generation settings (development-only; the +# server hard-ignores DEV_FAKE_GENERATION outside SERVER_ENVIRONMENT=Development). +function Get-DevStackKeysBlock { + $mediaRootForward = (Join-Path $DevStackDir "media") -replace '\\', '/' + return @" +# Fully-local dev stack (see _docs/dev_setup_local_stack.md): +# - DEV_FAKE_GENERATION: omni_gen image/video submits skip real providers and +# a resolver thread completes them a few seconds later with a placeholder +# result (or fails them when the prompt contains simulate_artcraft_failure). +# - CDN_BASE_URL: media URLs in API responses point at this server instead of +# the shared r2.dev bucket. Host only - no path suffix. +# - LOCAL_MEDIA_ROOT: directory served at /media (mirrors the bucket layout); +# seeded gallery media and fake generation results live here. +DEV_FAKE_GENERATION=true +CDN_BASE_URL=http://localhost:12345 +LOCAL_MEDIA_ROOT=$mediaRootForward +"@ +} + +# Bootstrap re-runs on a stack whose secrets file predates the fully-local +# media/fake-generation support: append the new keys (first-value-wins never +# lets these override anything the user already set). +function Add-DevStackKeysToExistingSecretsEnv { + $existing = Get-Content $SecretsEnvFile -Raw + if ($existing -match '(?m)^\s*LOCAL_MEDIA_ROOT=') { return } + Add-Content -Path $SecretsEnvFile -Value "`r`n$(Get-DevStackKeysBlock)" -Encoding ascii + Write-Log "Appended fully-local dev stack keys (DEV_FAKE_GENERATION, CDN_BASE_URL, LOCAL_MEDIA_ROOT) to $SecretsEnvFile" +} + function Build-Backend { if ($SkipRustBuild) { Write-Log "Skipped (-SkipRustBuild)." diff --git a/script/bootstrap/windows/common.ps1 b/script/bootstrap/windows/common.ps1 index a713f249e1..ea15ee0776 100644 --- a/script/bootstrap/windows/common.ps1 +++ b/script/bootstrap/windows/common.ps1 @@ -26,6 +26,11 @@ $script:DemoUsername = if ($env:DEMO_USERNAME) { $env:DEMO_USERNAME } else { "lo $script:DemoPassword = if ($env:DEMO_PASSWORD) { $env:DEMO_PASSWORD } else { "localdev1pass" } $script:DemoEmail = if ($env:DEMO_EMAIL) { $env:DEMO_EMAIL } else { "localdev1@example.com" } +# Banked credits the demo user's artcraft wallet is seeded/topped-up to. +# Generation submits are gated on wallet balance, so 0 credits means the +# generate button silently refuses in the webapp. +$script:DemoCredits = if ($env:DEMO_CREDITS) { [uint32]$env:DEMO_CREDITS } else { 100000 } + # --- Portable service pins ------------------------------------------------- # MySQL 8.4 LTS (the 8.x series is required; 9.x is unsupported by tooling). # cdn.mysql.com hosts only some patch versions; 8.4.11 verified available. @@ -48,6 +53,11 @@ $script:RedisDir = Join-Path $DevStackDir "redis" $script:LogsDir = Join-Path $DevStackDir "logs" $script:PidsDir = Join-Path $DevStackDir "pids" +# Local media served by the backend at /media when LOCAL_MEDIA_ROOT points +# here (fully-local gallery + fake generation results). Mirrors the public +# bucket layout: object path /media/{...} lives at .devstack\media\media\{...}. +$script:DevMediaRoot = Join-Path $DevStackDir "media" + $script:SecretsEnvFile = Join-Path $RootDir "crates\service\web\storyteller_web\config\storyteller-web.development-secrets.env" function Write-Log([string]$Message) { Write-Host "[bootstrap] $Message" } @@ -92,6 +102,16 @@ function Get-RedisTool([string]$Name) { return $null } +# Generate a token in the backend's format: {prefix}{crockford-lower entropy} +# padded to the same TOTAL length the Rust generators use (tokens crate, +# impl_crockford_generator!). Alphabet excludes i/l/o/u like Crockford base32. +function New-DevToken([string]$Prefix, [int]$TotalLength) { + $charset = "0123456789abcdefghjkmnpqrstvwxyz" + $count = $TotalLength - $Prefix.Length + $chars = 1..$count | ForEach-Object { $charset[(Get-Random -Maximum $charset.Length)] } + return "$Prefix$(-join $chars)" +} + # Run SQL as the app user against the dev database. Returns output lines # (stdout and stderr merged via Invoke-Native, so a caller capturing our # streams can never turn a mysql warning into a terminating error). diff --git a/script/bootstrap/windows/dev_stack_doctor.ps1 b/script/bootstrap/windows/dev_stack_doctor.ps1 index 74843eec23..379070d5ff 100644 --- a/script/bootstrap/windows/dev_stack_doctor.ps1 +++ b/script/bootstrap/windows/dev_stack_doctor.ps1 @@ -93,6 +93,16 @@ if (Test-TcpPort 3306) { $demoCount = Get-MySqlCount "SELECT COUNT(*) FROM users WHERE username='$DemoUsername'" if ($demoCount -gt 0) { Pass "demo user: '$DemoUsername' exists" } else { WarnCheck "demo user: '$DemoUsername' not created yet (seed_demo_user.ps1, backend must be running)" } + + if ($demoCount -gt 0) { + $credits = Get-MySqlCount "SELECT COALESCE(MAX(banked_credits + monthly_credits), 0) FROM wallets WHERE wallet_namespace='artcraft' AND owner_user_token=(SELECT token FROM users WHERE username='$DemoUsername')" + if ($credits -gt 0) { Pass "demo wallet: '$DemoUsername' has $credits credits" } + else { WarnCheck "demo wallet: '$DemoUsername' has no credits - generation flows are blocked (re-run seed_demo_user.ps1)" } + + $mediaCount = Get-MySqlCount "SELECT COUNT(*) FROM media_files WHERE maybe_creator_user_token=(SELECT token FROM users WHERE username='$DemoUsername')" + if ($mediaCount -gt 0) { Pass "demo media: '$DemoUsername' has $mediaCount media rows" } + else { WarnCheck "demo media: gallery is empty (re-run seed_demo_user.ps1)" } + } } else { Fail "mysql: port 3306 is up but '$DevMySqlUser' cannot access '$DevMySqlDb' (re-run bootstrap provisioning)" } @@ -113,6 +123,13 @@ Write-Host "--- Backend ---" if (Test-Path $SecretsEnvFile) { Pass "secrets env: storyteller-web.development-secrets.env exists" + + $secretsContent = Get-Content $SecretsEnvFile -Raw + if ($secretsContent -match '(?m)^\s*LOCAL_MEDIA_ROOT=') { + Pass "secrets env: fully-local dev stack keys present (DEV_FAKE_GENERATION, CDN_BASE_URL, LOCAL_MEDIA_ROOT)" + } else { + WarnCheck "secrets env: fully-local dev stack keys missing - generation/gallery use remote CDN and real providers (re-run bootstrap to append)" + } } else { Fail "secrets env: missing - the server aborts at boot on ~19 required vars (re-run bootstrap)" } diff --git a/script/bootstrap/windows/seed_demo_user.ps1 b/script/bootstrap/windows/seed_demo_user.ps1 index fba6a5e40e..c27f9fae22 100644 --- a/script/bootstrap/windows/seed_demo_user.ps1 +++ b/script/bootstrap/windows/seed_demo_user.ps1 @@ -51,7 +51,7 @@ $create = Invoke-JsonPost "$DevBackendUrl/v1/create_account" @{ if ($create.success -eq $true) { Write-Log "Account created." } else { - $errorCode = "$($create.error_code)$($create.error_code_str)" + $errorCode = "$($create.error_type)$($create.error_code)$($create.error_code_str)" if ($errorCode -match "(?i)username_?taken|email_?taken") { Write-Log "Account already exists ($errorCode) - continuing." } else { @@ -68,6 +68,98 @@ $login = Invoke-JsonPost "$DevBackendUrl/v1/login" @{ if ($login.success -ne $true) { Die "Login verification failed: $($login | ConvertTo-Json -Compress)" } if (-not $login.signed_session) { Die "Login succeeded but returned no signed_session." } +# --- Credits wallet --------------------------------------------------------- +# Generation submits deduct from the user's artcraft wallet and the webapp +# gates the generate button on the balance, so a user without credits cannot +# exercise any generation flow. Signup does not create a wallet; seed one. +# Raw SQL (not an endpoint): the only credit-granting paths are Stripe +# webhooks and moderator tooling, neither of which works locally. +Write-Log "Seeding credits wallet ($DemoCredits banked credits)..." + +$userToken = @(Invoke-MySqlApp -Sql "SELECT token FROM users WHERE username = '$DemoUsername' LIMIT 1") | + Where-Object { "$_" -match '^user_' } | Select-Object -First 1 +if (-not $userToken) { Die "Could not find users row for '$DemoUsername'." } + +$walletRow = @(Invoke-MySqlApp -Sql "SELECT token, banked_credits FROM wallets WHERE wallet_namespace = 'artcraft' AND owner_user_token = '$userToken' LIMIT 1") | + Where-Object { "$_" -match '^wallet_' } | Select-Object -First 1 + +if (-not $walletRow) { + $walletToken = New-DevToken "wallet_" 32 + $createLedgerToken = New-DevToken "wle_" 32 + $creditLedgerToken = New-DevToken "wle_" 32 + Invoke-MySqlApp -Sql ( + "INSERT INTO wallets SET token='$walletToken', wallet_namespace='artcraft', owner_user_token='$userToken', banked_credits=$DemoCredits, monthly_credits=0; " + + "INSERT INTO wallet_ledger_entries SET token='$createLedgerToken', wallet_token='$walletToken', entry_type='create', credits_delta=0, banked_credits_before=0, banked_credits_after=0, monthly_credits_before=0, monthly_credits_after=0; " + + "INSERT INTO wallet_ledger_entries SET token='$creditLedgerToken', wallet_token='$walletToken', entry_type='credit_banked', maybe_entity_ref='dev_bootstrap_seed', credits_delta=$DemoCredits, banked_credits_before=0, banked_credits_after=$DemoCredits, monthly_credits_before=0, monthly_credits_after=0;" + ) | Out-Null + if ($LASTEXITCODE -ne 0) { Die "Failed to insert the demo wallet." } + Write-Log "Wallet created with $DemoCredits banked credits." +} else { + $parts = "$walletRow" -split "`t" + $walletToken = $parts[0] + $balance = [uint64]$parts[1] + if ($balance -lt $DemoCredits) { + $creditLedgerToken = New-DevToken "wle_" 32 + $delta = [uint64]$DemoCredits - $balance + Invoke-MySqlApp -Sql ( + "UPDATE wallets SET banked_credits=$DemoCredits, version=version+1 WHERE token='$walletToken' LIMIT 1; " + + "INSERT INTO wallet_ledger_entries SET token='$creditLedgerToken', wallet_token='$walletToken', entry_type='credit_banked', maybe_entity_ref='dev_bootstrap_topup', credits_delta=$delta, banked_credits_before=$balance, banked_credits_after=$DemoCredits, monthly_credits_before=0, monthly_credits_after=0;" + ) | Out-Null + if ($LASTEXITCODE -ne 0) { Die "Failed to top up the demo wallet." } + Write-Log "Wallet topped up: $balance -> $DemoCredits banked credits." + } else { + Write-Log "Wallet already has $balance banked credits - leaving as-is." + } +} + +# --- Gallery media ---------------------------------------------------------- +# Seed a few images so the library/gallery isn't empty on a fresh stack. The +# files live under .devstack\media (served by the backend at /media when +# LOCAL_MEDIA_ROOT + CDN_BASE_URL are set - bootstrap writes both into the +# generated secrets env). Rows mirror what a real image upload would write. +Write-Log "Seeding demo gallery media..." + +$seedImages = @( + @{ Source = "test_data\image\juno.jpg"; Title = "Dev seed - Juno 1" }, + @{ Source = "test_data\image\mochi.jpg"; Title = "Dev seed - Mochi 1" }, + @{ Source = "test_data\image\juno.jpg"; Title = "Dev seed - Juno 2" }, + @{ Source = "test_data\image\mochi.jpg"; Title = "Dev seed - Mochi 2" }, + @{ Source = "test_data\image\juno.jpg"; Title = "Dev seed - Juno 3" }, + @{ Source = "test_data\image\mochi.jpg"; Title = "Dev seed - Mochi 3" } +) + +$existingMedia = Get-MySqlCount "SELECT COUNT(*) FROM media_files WHERE maybe_creator_user_token='$userToken' AND maybe_title LIKE 'Dev seed %'" +if ($existingMedia -ge $seedImages.Count) { + Write-Log "Gallery already has $existingMedia seeded media rows - leaving as-is." +} else { + foreach ($img in $seedImages) { + $source = Join-Path $RootDir $img.Source + if (-not (Test-Path $source)) { Die "Seed image missing: $source" } + + # Mirror MediaFileBucketPath: /media/{h0..h4}/{hash}/image_{hash}.jpg + $hash = New-DevToken "" 32 + $fanout = ($hash.Substring(0, 5).ToCharArray() -join "\") + $objectDir = Join-Path $DevMediaRoot "media\$fanout\$hash" + New-Item -ItemType Directory -Force $objectDir | Out-Null + Copy-Item $source (Join-Path $objectDir "image_$hash.jpg") -Force + + $sha = (Get-FileHash -Algorithm SHA256 $source).Hash.ToLower() + $size = (Get-Item $source).Length + $mediaToken = New-DevToken "m_" 32 + + Invoke-MySqlApp -Sql ( + "INSERT INTO media_files SET token='$mediaToken', maybe_creator_user_token='$userToken', " + + "origin_category='upload', origin_product_category='unknown', media_class='image', media_type='jpg', " + + "maybe_mime_type='image/jpeg', file_size_bytes=$size, checksum_sha2='$sha', " + + "public_bucket_directory_hash='$hash', maybe_public_bucket_prefix='image_', maybe_public_bucket_extension='.jpg', " + + "creator_ip_address='127.0.0.1', creator_set_visibility='public', is_user_upload=1, is_intermediate_system_file=0, " + + "maybe_title='$($img.Title)'" + ) | Out-Null + if ($LASTEXITCODE -ne 0) { Die "Failed to insert seeded media row for $($img.Title)." } + } + Write-Log "Seeded $($seedImages.Count) gallery images under $DevMediaRoot." +} + Write-Log "Demo user is ready." Write-Log " Username: $DemoUsername" Write-Log " Password: $DemoPassword"