improve concurrency

re-route request

.sqlx/query-d14ad3a64d261e64fa75f97c93ca74af97fad8591d6ebe9887aeabc43bed0982.json

@@ -1,6 +1,6 @@
 {
   "db_name": "MySQL",
-  "query": "\nSELECT\n    id,\n    token as `token: MediaFileToken`,\n    created_at as `created_at: DateTime<Utc>`,\n    maybe_thumbnail_version as `maybe_thumbnail_version: u8`,\n    public_bucket_directory_hash,\n    maybe_public_bucket_prefix,\n    maybe_public_bucket_extension\nFROM media_files\nWHERE\n    id >= (SELECT COALESCE(MIN(id), 0) FROM media_files WHERE created_at >= NOW() - INTERVAL ? HOUR)\n    AND id < ?\n    AND media_class = ?\n    AND maybe_thumbnail_version IS NULL\n    AND user_deleted_at IS NULL\n    AND mod_deleted_at IS NULL\nORDER BY id DESC\nLIMIT ?\n    ",
+  "query": "\nSELECT\n    media.id,\n    media.token as `token: MediaFileToken`,\n    media.created_at as `created_at: DateTime<Utc>`,\n    NOW(6) as `database_read_at!: DateTime<Utc>`,\n    jobs.successfully_completed_at as `maybe_generation_completed_at?: DateTime<Utc>`,\n    media.maybe_thumbnail_version as `maybe_thumbnail_version: u8`,\n    media.public_bucket_directory_hash,\n    media.maybe_public_bucket_prefix,\n    media.maybe_public_bucket_extension\nFROM media_files AS media\nLEFT JOIN generic_inference_jobs AS jobs ON jobs.token = media.maybe_source_job_token\nWHERE\n    media.id >= (SELECT COALESCE(MIN(id), 0) FROM media_files WHERE created_at >= NOW() - INTERVAL ? HOUR)\n    AND media.id < ?\n    AND media.media_class = ?\n    AND media.maybe_thumbnail_version IS NULL\n    AND media.user_deleted_at IS NULL\n    AND media.mod_deleted_at IS NULL\nORDER BY media.id DESC\nLIMIT ?\n    ",
   "describe": {
     "columns": [
       {
@@ -35,6 +35,26 @@
       },
       {
         "ordinal": 3,
+        "name": "database_read_at!: DateTime<Utc>",
+        "type_info": {
+          "type": "Datetime",
+          "flags": "NOT_NULL | BINARY",
+          "char_set": 63,
+          "max_size": 26
+        }
+      },
+      {
+        "ordinal": 4,
+        "name": "maybe_generation_completed_at?: DateTime<Utc>",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "BINARY",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 5,
         "name": "maybe_thumbnail_version: u8",
         "type_info": {
           "type": "Tiny",
@@ -44,7 +64,7 @@
         }
       },
       {
-        "ordinal": 4,
+        "ordinal": 6,
         "name": "public_bucket_directory_hash",
         "type_info": {
           "type": "VarString",
@@ -54,7 +74,7 @@
         }
       },
       {
-        "ordinal": 5,
+        "ordinal": 7,
         "name": "maybe_public_bucket_prefix",
         "type_info": {
           "type": "VarString",
@@ -64,7 +84,7 @@
         }
       },
       {
-        "ordinal": 6,
+        "ordinal": 8,
         "name": "maybe_public_bucket_extension",
         "type_info": {
           "type": "VarString",
@@ -81,11 +101,13 @@
       false,
       false,
       false,
+      false,
+      true,
       true,
       false,
       true,
       true
     ]
   },
-  "hash": "daed2dcd03b3c9fbdb9057b743df078f43e2af87a2c31f418234f98da5ff979f"
+  "hash": "d14ad3a64d261e64fa75f97c93ca74af97fad8591d6ebe9887aeabc43bed0982"
 }

Cargo.lock

@@ -10384,11 +10384,13 @@ dependencies = [
  "anyhow",
  "bootstrap",
  "bucket_paths",
+ "chrono",
  "cloud_storage",
  "concurrency",
  "easyenv",
  "errors",
  "ffmpeg_utils",
+ "futures 0.3.32",
  "jobs_common",
  "log",
  "mysql_queries",

crates/lib/ffmpeg_utils/src/ffmpeg/ffmpeg_video_first_frame_to_jpg_thumbnail.rs

@@ -1,11 +1,11 @@
-use anyhow::bail;
-use log::{error, info};
-use std::io::Write;
+use log::info;
 use std::path::Path;
 use std::process::Command;
 
 use errors::AnyhowResult;
 
+use crate::ffmpeg::run_thumbnail_ffmpeg::run_thumbnail_ffmpeg;
+
 pub struct FfmpegVideoFirstFrameToJpgThumbnailArgs<I: AsRef<Path>, O: AsRef<Path>> {
   pub input_video_path: I,
   pub output_jpg_path: O,
@@ -24,26 +24,20 @@ pub fn ffmpeg_video_first_frame_to_jpg_thumbnail<I: AsRef<Path>, O: AsRef<Path>>
   command
       .arg("-nostdin")
       .arg("-y")
+      .arg("-loglevel").arg("error")
+      .arg("-nostats")
+      .arg("-threads").arg("1")
+      .arg("-filter_threads").arg("1")
       .arg("-sseof").arg("-1")
       .arg("-i").arg(args.input_video_path.as_ref())
+      .arg("-threads").arg("1")
       .arg("-vf").arg("scale=320:320:force_original_aspect_ratio=decrease")
       .arg("-vframes").arg("1")
       .arg(args.output_jpg_path.as_ref());
 
   info!("Calling ffmpeg (jpg thumbnail)...");
 
-  let output = command.output()?;
-
-  if !output.status.success() {
-    error!("bad exit status: {}", output.status);
-
-    let _r = std::io::stdout().write_all(&output.stdout);
-    let _r = std::io::stderr().write_all(&output.stderr);
-
-    bail!("ffmpeg jpg thumbnail failed: {:?}", output.status.to_string());
-  }
-
-  Ok(())
+  run_thumbnail_ffmpeg(command)
 }
 
 #[cfg(test)]

crates/lib/ffmpeg_utils/src/ffmpeg/ffmpeg_video_gif_preview.rs

@@ -1,11 +1,11 @@
-use anyhow::bail;
-use log::{error, info};
-use std::io::Write;
+use log::info;
 use std::path::Path;
 use std::process::Command;
 
 use errors::AnyhowResult;
 
+use crate::ffmpeg::run_thumbnail_ffmpeg::run_thumbnail_ffmpeg;
+
 pub struct FfmpegVideoGifPreviewArgs<I: AsRef<Path>, O: AsRef<Path>> {
   pub input_video_path: I,
   pub output_gif_path: O,
@@ -13,7 +13,7 @@ pub struct FfmpegVideoGifPreviewArgs<I: AsRef<Path>, O: AsRef<Path>> {
 
 /// Extract a short animated GIF preview from a video file.
 ///
-/// Takes the first 5 seconds, resampled to 10 fps, scaled to 360px wide
+/// Takes the first 5 seconds, resampled to 10 fps, scaled to fit within 360x360
 /// (preserving aspect ratio), with an optimized color palette.
 pub fn ffmpeg_video_gif_preview<I: AsRef<Path>, O: AsRef<Path>>(
   args: FfmpegVideoGifPreviewArgs<I, O>,
@@ -23,27 +23,22 @@ pub fn ffmpeg_video_gif_preview<I: AsRef<Path>, O: AsRef<Path>>(
   command
       .arg("-nostdin")
       .arg("-y")
+      .arg("-loglevel").arg("error")
+      .arg("-nostats")
+      .arg("-threads").arg("1")
+      .arg("-filter_complex_threads").arg("1")
       .arg("-ss").arg("0")
       .arg("-to").arg("5")
       .arg("-i").arg(args.input_video_path.as_ref())
+      .arg("-threads").arg("1")
       .arg("-filter_complex")
-      .arg("fps=10,scale=360:-1[s]; [s]split[a][b]; [a]palettegen[palette]; [b][palette]paletteuse")
+      .arg("fps=10,scale=360:360:force_original_aspect_ratio=decrease[s]; [s]split[a][b]; [a]palettegen[palette]; [b][palette]paletteuse")
+      .arg("-frames:v").arg("50")
       .arg(args.output_gif_path.as_ref());
 
   info!("Calling ffmpeg (gif preview)...");
 
-  let output = command.output()?;
-
-  if !output.status.success() {
-    error!("bad exit status: {}", output.status);
-
-    let _r = std::io::stdout().write_all(&output.stdout);
-    let _r = std::io::stderr().write_all(&output.stderr);
-
-    bail!("ffmpeg gif preview failed: {:?}", output.status.to_string());
-  }
-
-  Ok(())
+  run_thumbnail_ffmpeg(command)
 }
 
 #[cfg(test)]
@@ -69,7 +64,7 @@ mod tests {
 
     assert!(output_path.exists(), "output gif should exist");
 
-    // The input video is 640x480. scale=360:-1 should produce 360x270
+    // The input video is 640x480. Fitting within 360x360 should produce 360x270
     // (maintaining 4:3 aspect ratio).
     // Use ffprobe to verify the GIF dimensions.
     let result = ffprobe::ffprobe(&output_path)
@@ -82,4 +77,30 @@ mod tests {
     assert_eq!(video_stream.width, Some(360));
     assert_eq!(video_stream.height, Some(270));
   }
+
+  #[test]
+  fn test_tall_video_preview_has_bounded_dimensions_and_frame_count() {
+    let temp_dir = TempDir::new_in("/tmp", "ffmpeg_tall_gif_test").unwrap();
+    let input = temp_dir.path().join("tall.mp4");
+    let output = temp_dir.path().join("preview.gif");
+    let status = Command::new("ffmpeg")
+      .args(["-nostdin", "-y", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=size=32x640:rate=10", "-t", "6", "-threads", "1"])
+      .arg(&input)
+      .status().unwrap();
+    assert!(status.success());
+
+    ffmpeg_video_gif_preview(FfmpegVideoGifPreviewArgs {
+      input_video_path: &input,
+      output_gif_path: &output,
+    }).unwrap();
+    let probe = Command::new("ffprobe")
+      .args(["-v", "error", "-count_frames", "-select_streams", "v:0", "-show_entries", "stream=width,height,nb_read_frames", "-of", "csv=p=0"])
+      .arg(&output)
+      .output().unwrap();
+    assert!(probe.status.success());
+    let values: Vec<u32> = std::str::from_utf8(&probe.stdout).unwrap().trim()
+      .split(',').map(|value| value.parse().unwrap()).collect();
+    assert_eq!(&values[..2], &[18, 360]);
+    assert!(values[2] > 0 && values[2] <= 50);
+  }
 }

crates/lib/ffmpeg_utils/src/ffmpeg/mod.rs

@@ -3,3 +3,4 @@ pub mod ffmpeg_transcode_to_mp4;
 pub mod ffmpeg_trim_and_resample;
 pub mod ffmpeg_video_first_frame_to_jpg_thumbnail;
 pub mod ffmpeg_video_gif_preview;
+mod run_thumbnail_ffmpeg;

crates/lib/ffmpeg_utils/src/ffmpeg/run_thumbnail_ffmpeg.rs

@@ -0,0 +1,75 @@
+use std::io::{Read, Seek, SeekFrom};
+use std::process::{Child, Command, ExitStatus, Stdio};
+use std::thread;
+use std::time::{Duration, Instant};
+
+use anyhow::bail;
+use log::error;
+
+use errors::AnyhowResult;
+
+const THUMBNAIL_TIMEOUT: Duration = Duration::from_secs(60);
+const MAX_ERROR_LOG_BYTES: u64 = 16 * 1024;
+
+/// Synchronous: callers in async runtimes must use spawn_blocking. A timeout
+/// kills and reaps FFmpeg before returning, so it cannot outlive its job slot.
+pub(super) fn run_thumbnail_ffmpeg(mut command: Command) -> AnyhowResult<()> {
+  // Avoid holding arbitrarily large FFmpeg stderr output in the Rust heap.
+  let mut stderr = tempfile::tempfile()?;
+  let mut child = command.stdout(Stdio::null()).stderr(Stdio::from(stderr.try_clone()?)).spawn()?;
+  let result = wait_for_exit(&mut child, THUMBNAIL_TIMEOUT);
+  if !matches!(&result, Ok(status) if status.success()) {
+    let start = stderr.metadata()?.len().saturating_sub(MAX_ERROR_LOG_BYTES);
+    stderr.seek(SeekFrom::Start(start))?;
+    let mut bytes = Vec::new();
+    stderr.take(MAX_ERROR_LOG_BYTES).read_to_end(&mut bytes)?;
+    error!("Thumbnail FFmpeg failed: {:?}; stderr: {}", result, String::from_utf8_lossy(&bytes));
+  }
+  let status = result?;
+  if !status.success() {
+    bail!("thumbnail FFmpeg failed: {status}");
+  }
+  Ok(())
+}
+
+fn wait_for_exit(child: &mut Child, timeout: Duration) -> AnyhowResult<ExitStatus> {
+  let started_at = Instant::now();
+  loop {
+    match child.try_wait() {
+      Ok(Some(status)) => return Ok(status),
+      Ok(None) => {},
+      Err(err) => {
+        let _ = child.kill();
+        let _ = child.wait();
+        return Err(err.into());
+      }
+    }
+    if started_at.elapsed() >= timeout {
+      let _ = child.kill();
+      child.wait()?;
+      bail!("thumbnail FFmpeg exceeded timeout of {} seconds", timeout.as_secs_f64());
+    }
+    thread::sleep(Duration::from_millis(25));
+  }
+}
+
+#[cfg(all(test, unix))]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn timed_out_child_is_killed_and_reaped_before_returning() {
+    let mut child = Command::new("sleep").arg("10").spawn().unwrap();
+    let started_at = Instant::now();
+    let result = wait_for_exit(&mut child, Duration::from_millis(50));
+    assert!(result.unwrap_err().to_string().contains("exceeded timeout"));
+    assert!(started_at.elapsed() < Duration::from_secs(3));
+    assert!(!child.try_wait().unwrap().unwrap().success());
+  }
+
+  #[test]
+  fn reports_normal_child_exit_status() {
+    let mut child = Command::new("true").spawn().unwrap();
+    assert!(wait_for_exit(&mut child, Duration::from_secs(3)).unwrap().success());
+  }
+}

crates/schema/database/mysql_queries/src/queries/media_files/thumbnails/list_video_media_files_without_thumbnails_for_job.rs

@@ -27,6 +27,11 @@ pub struct VideoMediaFileWithoutThumbnail {
   pub id: i64,
   pub token: MediaFileToken,
   pub created_at: DateTime<Utc>,
+  /// Database statement-start time, shared by every row in this page.
+  pub database_read_at: DateTime<Utc>,
+  /// Our job's successful finalization time, not the provider's completion time.
+  /// Uploads, legacy records, and jobs still being finalized can have no timestamp.
+  pub maybe_generation_completed_at: Option<DateTime<Utc>>,
   pub maybe_thumbnail_version: Option<u8>,
   pub public_bucket_directory_hash: String,
   pub maybe_public_bucket_prefix: Option<String>,
@@ -51,22 +56,25 @@ pub async fn list_video_media_files_without_thumbnails_for_job<'e, 'c, E>(
     VideoMediaFileWithoutThumbnail,
     r#"
 SELECT
-    id,
-    token as `token: MediaFileToken`,
-    created_at as `created_at: DateTime<Utc>`,
-    maybe_thumbnail_version as `maybe_thumbnail_version: u8`,
-    public_bucket_directory_hash,
-    maybe_public_bucket_prefix,
-    maybe_public_bucket_extension
-FROM media_files
+    media.id,
+    media.token as `token: MediaFileToken`,
+    media.created_at as `created_at: DateTime<Utc>`,
+    NOW(6) as `database_read_at!: DateTime<Utc>`,
+    jobs.successfully_completed_at as `maybe_generation_completed_at?: DateTime<Utc>`,
+    media.maybe_thumbnail_version as `maybe_thumbnail_version: u8`,
+    media.public_bucket_directory_hash,
+    media.maybe_public_bucket_prefix,
+    media.maybe_public_bucket_extension
+FROM media_files AS media
+LEFT JOIN generic_inference_jobs AS jobs ON jobs.token = media.maybe_source_job_token
 WHERE
-    id >= (SELECT COALESCE(MIN(id), 0) FROM media_files WHERE created_at >= NOW() - INTERVAL ? HOUR)
-    AND id < ?
-    AND media_class = ?
-    AND maybe_thumbnail_version IS NULL
-    AND user_deleted_at IS NULL
-    AND mod_deleted_at IS NULL
-ORDER BY id DESC
+    media.id >= (SELECT COALESCE(MIN(id), 0) FROM media_files WHERE created_at >= NOW() - INTERVAL ? HOUR)
+    AND media.id < ?
+    AND media.media_class = ?
+    AND media.maybe_thumbnail_version IS NULL
+    AND media.user_deleted_at IS NULL
+    AND media.mod_deleted_at IS NULL
+ORDER BY media.id DESC
 LIMIT ?
     "#,
     max_lookback_hours,

crates/service/job/video_thumbnail_job/Cargo.toml

@@ -39,6 +39,8 @@ server_environment = { path = "../../../lib/server_environment" }
 # External
 actix-web.workspace = true
 anyhow.workspace = true
+chrono.workspace = true
+futures.workspace = true
 log.workspace = true
 serde.workspace = true
 serde_derive.workspace = true

crates/service/job/video_thumbnail_job/README.md

@@ -0,0 +1,102 @@
+# Video thumbnail worker
+
+## Scheduling and resource settings
+
+| Environment variable                       | Default | Meaning                                      |
+| ------------------------------------------ | ------- | -------------------------------------------- |
+| VIDEO_THUMBNAIL_CONCURRENCY                | 1       | Maximum simultaneous complete file attempts  |
+| VIDEO_THUMBNAIL_POLL_INTERVAL_MILLIS       | 5000    | Idle polling interval                        |
+| VIDEO_THUMBNAIL_RETRY_CACHE_CAPACITY       | 10000   | Maximum cached failed media IDs              |
+| VIDEO_THUMBNAIL_RETRY_INITIAL_DELAY_MILLIS | 30000   | Backoff after the first failed attempt       |
+| VIDEO_THUMBNAIL_RETRY_MAX_DELAY_MILLIS     | 1800000 | Maximum backoff (configurable up to one day) |
+
+Failures back off for 30 seconds, 60 seconds, 120 seconds, and so on, capped at
+30 minutes by default. Successful retries clear their failure history. The
+cache retains only IDs, attempt counts, and monotonic timestamps. At capacity,
+the least recently failed entry is evicted. Restarts and eviction reset that
+entry's history; no failure state is persisted in the database. Invalid files
+are periodically retried rather than permanently marked failed.
+
+Files in backoff are skipped, and pagination advances even across pages made
+entirely of skipped files. Within each page, untried files precede due retries.
+The worker starts at most the configured number of attempts, admitting another
+as soon as a slot is free. The concurrency limit covers downloads, processing,
+uploads, and the database update. Raising it does not coordinate other pods;
+multiple pods still require distinct shard assignments.
+
+FFmpeg runs on Tokio's blocking pool, one process at a time per file. Each
+process uses one codec thread and one filter thread, and is killed and reaped
+after 60 seconds if it has not finished. Stderr is spooled to a temporary file;
+only its final 16 KiB is read into memory on failure. GIF previews fit within
+360x360 and contain at most 50 frames (five seconds at ten frames per second).
+Portrait GIFs are consequently smaller than the former 360px-wide previews.
+
+These controls reduce memory use but do not guarantee an OOM-free decode:
+source resolution, codec reference frames, and multiple concurrent files still
+matter. Keep concurrency at one initially. With the previously observed
+approximately 560 MiB container peak, two jobs are not proven safe under a 1 GiB
+memory limit. Measure representative large inputs before raising concurrency.
+
+On cgroup v2, `thumbnail_memory` logs container `current_bytes`, `peak_bytes`,
+and `limit_bytes` every ten seconds, including FFmpeg children and charged file
+cache. Usage at or above 80% of the limit is logged as a warning. The peak is a
+container lifetime high-water mark, not a per-file allocation. Memory reporting
+is advisory; it does not pause admission or replace the container memory limit.
+Unavailable metrics (including on macOS) are reported without blocking work.
+
+## Timing
+
+The worker emits `thumbnail_timing` followed by a JSON object at the start and
+end of every attempt. Filter by `media_file_token` to correlate events.
+
+- `event=started`: before downloading the source video.
+- `event=completed`: JPG and GIF uploaded and the thumbnail version updated in
+  the database, so the thumbnails are available to API readers.
+- `event=failed`: an attempt ended unsuccessfully; exclude this from successful
+  thumbnail latency percentiles.
+
+`since_generation_completed_ms` measures time from
+`generic_inference_jobs.successfully_completed_at` to the event. The source job
+is found through `media_files.maybe_source_job_token`. This timestamp records
+our backend finalizing the generation job after ingesting its output. It does
+not measure when the external provider originally finished generating video,
+nor is it an exact transaction commit timestamp.
+
+For uploads, older unlinked media, or a job whose completion has not yet been
+written when the page is fetched, `generation_completed_at` and
+`since_generation_completed_ms` are JSON `null`. They are never replaced with
+media creation time. `since_media_created_ms` is reported separately for all
+files. `media_files.updated_at` is unsuitable because subsequent edits and the
+thumbnail update change it.
+
+Additional measurements:
+
+- `processing_elapsed_ms`: monotonic time spent on this attempt; zero at start.
+- `page_wait_ms`: time from fetching the page until this file starts, including
+  waiting for earlier files in the same page.
+- `database_query_ms`: total time spent fetching the page, including pool wait.
+- `thumbnail_stage`: JSON events with `stage`, `success`, and `duration_ms` for
+  download, JPG generation/upload, GIF generation/upload, and database update.
+- `thumbnail_page`: fetched row count and query duration, including empty pages.
+
+## Clock handling
+
+The listing query returns `NOW(6)` as `database_read_at` with each row. Both that
+timestamp and the generation completion timestamp come from the database.
+Worker wall-clock time is not used to calculate latency.
+
+MySQL evaluates `NOW(6)` at statement start. We bracket the query with two Rust
+`Instant`s and use their midpoint as the estimated local instant corresponding
+to `database_read_at`. Every event advances this database time using monotonic
+elapsed time, including time queued behind other files in the page.
+
+This estimate has up to `clock_uncertainty_ms` of error in either direction
+(half the query duration, rounded up), plus the source timestamps' existing
+one-second precision. For example, a 20 ms query contributes up to 10 ms of
+clock-mapping uncertainty. A slow query reports a larger uncertainty; do not
+interpret millisecond-valued fields as millisecond accuracy. Negative values
+are preserved to expose timestamp anomalies. No extra per-file clock query is
+needed.
+
+These measurements end at backend thumbnail availability. Client polling,
+caching, and UI refresh delays happen afterward and need separate measurement.

crates/service/job/video_thumbnail_job/config/video-thumbnail-job.development.env

@@ -5,3 +5,7 @@ SERVER_ENVIRONMENT='development'
 
 # Don't collide with storyteller-web in development
 HTTP_BIND_ADDRESS=0.0.0.0:13339
+
+# Limit whole thumbnail attempts, including FFmpeg and downloads.
+VIDEO_THUMBNAIL_CONCURRENCY=1
+VIDEO_THUMBNAIL_POLL_INTERVAL_MILLIS=5000

crates/service/job/video_thumbnail_job/src/job/main_loop.rs

@@ -1,19 +1,30 @@
-use std::time::Duration;
+use std::future::Future;
+use std::num::NonZeroUsize;
+use std::time::{Duration, Instant};
 
+use futures::{stream, StreamExt};
 use log::{error, info, warn};
 
 use mysql_queries::queries::media_files::thumbnails::list_video_media_files_without_thumbnails_for_job::{
   list_video_media_files_without_thumbnails_for_job,
   ListVideoMediaFilesWithoutThumbnailsArgs,
+  VideoMediaFileWithoutThumbnail,
 };
 
 use crate::job::alert_on_error::alert_pager_and_return_err;
 use crate::job_dependencies::JobDependencies;
 use crate::job::process_single_media_file::process_single_media_file;
+use crate::job::thumbnail_timing::ThumbnailTiming;
+use crate::job::memory_monitor::monitor_memory;
+use crate::job::retry_backoff::RetryBackoff;
 
 pub async fn main_loop(deps: JobDependencies) {
+  let config = &deps.worker_config;
+  let mut retry_cache = RetryBackoff::new(config.retry_cache_capacity, config.retry_initial_delay, config.retry_max_delay);
+  let memory_monitor = tokio::spawn(monitor_memory(config.concurrency.get()));
+
   while !deps.application_shutdown.get() {
-    let processed_any = match run_batch_cycle(&deps).await {
+    let processed_any = match run_batch_cycle(&deps, &mut retry_cache).await {
       Ok(count) => {
         if count > 0 {
           info!("Processed {} video thumbnail(s) this cycle.", count);
@@ -37,12 +48,13 @@ pub async fn main_loop(deps: JobDependencies) {
     }
   }
 
+  memory_monitor.abort();
   warn!("Video thumbnail job main loop is shut down.");
 }
 
 /// Run one full pagination cycle: keep querying pages of media files without thumbnails
 /// until there are no more results. Returns the total number of items processed.
-async fn run_batch_cycle(deps: &JobDependencies) -> anyhow::Result<u64> {
+async fn run_batch_cycle(deps: &JobDependencies, retry_cache: &mut RetryBackoff) -> anyhow::Result<u64> {
   let mut maybe_cursor: Option<i64> = None;
   let mut total_processed: u64 = 0;
 
@@ -51,6 +63,7 @@ async fn run_batch_cycle(deps: &JobDependencies) -> anyhow::Result<u64> {
       break;
     }
 
+    let query_started_at = Instant::now();
     let result = list_video_media_files_without_thumbnails_for_job(
       ListVideoMediaFilesWithoutThumbnailsArgs {
         custom_max_lookback_hours: deps.custom_max_lookback_hours,
@@ -59,41 +72,43 @@ async fn run_batch_cycle(deps: &JobDependencies) -> anyhow::Result<u64> {
         executor: &deps.mysql_pool,
       },
     ).await?;
+    let query_finished_at = Instant::now();
+
+    info!(
+      "thumbnail_page fetched_count={} database_query_ms={}",
+      result.media_files.len(),
+      query_finished_at.duration_since(query_started_at).as_millis(),
+    );
 
     if result.media_files.is_empty() {
       break;
     }
 
-    for media_file in &result.media_files {
-      if deps.application_shutdown.get() {
-        break;
-      }
-
-      // If sharding is enabled, skip media files that don't belong to this shard.
-      if let Some(shard_info) = &deps.shard_info {
-        let modulus = (media_file.id as u64) % (shard_info.number_of_shards as u64);
-        if modulus != shard_info.shard_index as u64 {
-          continue;
-        }
-      }
-
-      match process_single_media_file(deps, media_file).await {
-        Ok(()) => {
-          let _ = deps.job_stats.increment_success_count();
-          total_processed += 1;
-        }
-        Err(err) => {
-          warn!(
-            "Failed to generate thumbnail for media file {}: {:?}",
-            media_file.token.as_str(),
-            err,
-          );
-          let _ = deps.job_stats.increment_failure_count();
-        }
-      }
-    }
-
+    // Advance using the database page, even when every row is in backoff or
+    // belongs to another shard. Otherwise failed files could hide older good files.
     maybe_cursor = result.next_cursor;
+    let media_files = result.media_files.into_iter().filter(|media_file| {
+      deps.shard_info.as_ref().is_none_or(|shard| {
+        (media_file.id as u64) % u64::from(shard.number_of_shards) == u64::from(shard.shard_index)
+      })
+    }).collect();
+
+    total_processed += process_page(media_files, deps.worker_config.concurrency, retry_cache, |media_file| async move {
+      // Already-running attempts are drained on shutdown; no new attempts start.
+      if deps.application_shutdown.get() {
+        return None;
+      }
+      let timing = ThumbnailTiming::start(&media_file, query_started_at, query_finished_at);
+      let result = process_single_media_file(deps, &media_file, &timing).await;
+      timing.finish(result.is_ok());
+      if result.is_ok() {
+        let _ = deps.job_stats.increment_success_count();
+      } else {
+        let _ = deps.job_stats.increment_failure_count();
+      }
+      Some(result)
+    }).await;
+
     if maybe_cursor.is_none() {
       break;
     }
@@ -104,3 +119,141 @@ async fn run_batch_cycle(deps: &JobDependencies) -> anyhow::Result<u64> {
 
   Ok(total_processed)
 }
+
+/// Only keep `concurrency` attempts in flight, including their download, FFmpeg,
+/// and upload work. Complete one before admitting another; never spawn a page of
+/// tasks that merely wait for a semaphore while retaining downloaded videos.
+async fn process_page<F, Fut>(
+  mut media_files: Vec<VideoMediaFileWithoutThumbnail>,
+  concurrency: NonZeroUsize,
+  retry_cache: &mut RetryBackoff,
+  process: F,
+) -> u64
+where
+  F: Fn(VideoMediaFileWithoutThumbnail) -> Fut,
+  Fut: Future<Output = Option<anyhow::Result<()>>>,
+{
+  let fetched_count = media_files.len();
+  let now = Instant::now();
+  media_files.retain(|file| retry_cache.is_ready(file.id, now));
+  // Keep database ordering within each group, but let untried files precede retries.
+  media_files.sort_by_key(|file| retry_cache.has_failed(file.id));
+  info!(
+    "thumbnail_dispatch eligible_count={} backoff_skipped_count={} concurrency={}",
+    media_files.len(), fetched_count - media_files.len(), concurrency,
+  );
+
+  let mut pending = stream::iter(media_files).map(|file| {
+    let id = file.id;
+    let token = file.token.clone();
+    let future = process(file);
+    async move { (id, token, future.await) }
+  }).buffer_unordered(concurrency.get());
+  let mut completed = 0;
+  while let Some((id, token, maybe_result)) = pending.next().await {
+    match maybe_result {
+      Some(Ok(())) => {
+        retry_cache.record_success(id);
+        completed += 1;
+      }
+      Some(Err(err)) => {
+        let retry = retry_cache.record_failure(id, Instant::now());
+        warn!(
+          "thumbnail_retry media_file_token={} failed_attempts={} retry_delay_ms={} error={:?}",
+          token.as_str(), retry.attempts, retry.delay.as_millis(), err,
+        );
+      }
+      None => {}, // Shutdown: this file was not attempted.
+    }
+  }
+  completed
+}
+
+#[cfg(test)]
+mod tests {
+  use std::sync::atomic::{AtomicUsize, Ordering};
+  use std::sync::Mutex;
+
+  use chrono::DateTime;
+  use serde_json::json;
+
+  use super::*;
+
+  mod dispatch {
+    use super::*;
+
+    #[tokio::test]
+    async fn fills_available_slots_without_exceeding_configured_concurrency() {
+      for concurrency in 1..=4 {
+        let active = AtomicUsize::new(0);
+        let peak = AtomicUsize::new(0);
+        let mut cache = retry_cache();
+        let completed = process_page((1..=20).map(media_file).collect(), limit(concurrency), &mut cache, |_| async {
+          let count = active.fetch_add(1, Ordering::SeqCst) + 1;
+          peak.fetch_max(count, Ordering::SeqCst);
+          tokio::task::yield_now().await;
+          active.fetch_sub(1, Ordering::SeqCst);
+          Some(Ok(()))
+        }).await;
+        assert_eq!(completed, 20);
+        assert_eq!(peak.load(Ordering::SeqCst), concurrency);
+        assert_eq!(active.load(Ordering::SeqCst), 0);
+      }
+    }
+
+    #[tokio::test]
+    async fn cached_failure_page_does_not_prevent_processing_a_later_good_page() {
+      let mut cache = retry_cache();
+      for id in 1..=100 {
+        cache.record_failure(id, Instant::now());
+      }
+      let completed = process_page((1..=100).map(media_file).collect(), limit(2), &mut cache, |_| async {
+        panic!("A file in backoff must not run");
+      }).await;
+      assert_eq!(completed, 0);
+      let completed = process_page(vec![media_file(101)], limit(2), &mut cache, |_| async { Some(Ok(())) }).await;
+      assert_eq!(completed, 1);
+    }
+
+    #[tokio::test]
+    async fn untried_files_precede_due_retries_and_failure_does_not_stop_progress() {
+      let mut cache = retry_cache();
+      cache.record_failure(1, Instant::now() - Duration::from_secs(60));
+      let order = Mutex::new(Vec::new());
+      let completed = process_page(vec![media_file(1), media_file(2)], limit(1), &mut cache, |file| {
+        let order = &order;
+        async move {
+          order.lock().unwrap().push(file.id);
+          Some(if file.id == 1 { Err(anyhow::anyhow!("invalid video")) } else { Ok(()) })
+        }
+      }).await;
+      assert_eq!(completed, 1);
+      assert_eq!(*order.lock().unwrap(), vec![2, 1]);
+      assert!(!cache.is_ready(1, Instant::now()));
+      assert!(!cache.has_failed(2));
+    }
+  }
+
+  fn retry_cache() -> RetryBackoff {
+    RetryBackoff::new(limit(1000), Duration::from_secs(30), Duration::from_secs(1800))
+  }
+
+  fn limit(value: usize) -> NonZeroUsize {
+    NonZeroUsize::new(value).unwrap()
+  }
+
+  fn media_file(id: i64) -> VideoMediaFileWithoutThumbnail {
+    let now = DateTime::from_timestamp(1_000_000, 0).unwrap();
+    VideoMediaFileWithoutThumbnail {
+      id,
+      token: serde_json::from_value(json!(format!("m_test_{id}"))).unwrap(),
+      created_at: now,
+      database_read_at: now,
+      maybe_generation_completed_at: Some(now),
+      maybe_thumbnail_version: None,
+      public_bucket_directory_hash: String::new(),
+      maybe_public_bucket_prefix: None,
+      maybe_public_bucket_extension: None,
+    }
+  }
+}

crates/service/job/video_thumbnail_job/src/job/memory_monitor.rs

@@ -0,0 +1,73 @@
+use std::path::Path;
+use std::time::Duration;
+
+use log::{info, warn};
+use serde_json::json;
+use tokio::fs;
+
+struct MemoryUsage {
+  current_bytes: u64,
+  maybe_peak_bytes: Option<u64>,
+  maybe_limit_bytes: Option<u64>,
+}
+
+/// cgroup v2 accounts for the Rust worker, FFmpeg children, and charged file cache.
+pub async fn monitor_memory(concurrency: usize) {
+  let root = Path::new("/sys/fs/cgroup");
+  let mut interval = tokio::time::interval(Duration::from_secs(10));
+  loop {
+    interval.tick().await;
+    let Some(usage) = read_memory_usage(root).await else {
+      info!("Thumbnail container memory measurements unavailable (requires readable cgroup v2 memory files)");
+      return;
+    };
+    let fields = json!({
+      "configured_concurrency": concurrency,
+      "current_bytes": usage.current_bytes,
+      "peak_bytes": usage.maybe_peak_bytes,
+      "limit_bytes": usage.maybe_limit_bytes,
+    });
+    if usage.maybe_limit_bytes.is_some_and(|limit| usage.current_bytes >= limit.saturating_mul(4) / 5) {
+      warn!("thumbnail_memory {}", fields);
+    } else {
+      info!("thumbnail_memory {}", fields);
+    }
+  }
+}
+
+async fn read_memory_usage(root: &Path) -> Option<MemoryUsage> {
+  let current_bytes = read_number(&root.join("memory.current")).await?;
+  let maybe_peak_bytes = read_number(&root.join("memory.peak")).await;
+  let maybe_limit_bytes = read_number(&root.join("memory.max")).await;
+  Some(MemoryUsage { current_bytes, maybe_peak_bytes, maybe_limit_bytes })
+}
+
+async fn read_number(path: &Path) -> Option<u64> {
+  // "max" denotes no cgroup memory limit; old kernels may lack memory.peak.
+  fs::read_to_string(path).await.ok()?.trim().parse().ok()
+}
+
+#[cfg(test)]
+mod tests {
+  use tempdir::TempDir;
+
+  use super::*;
+
+  #[tokio::test]
+  async fn reads_container_usage_and_handles_unlimited_or_missing_optional_metrics() {
+    let directory = TempDir::new("thumbnail_memory_test").unwrap();
+    let root = directory.path();
+    fs::write(root.join("memory.current"), "1234\n").await.unwrap();
+    fs::write(root.join("memory.max"), "max\n").await.unwrap();
+    let usage = read_memory_usage(root).await.unwrap();
+    assert_eq!(usage.current_bytes, 1234);
+    assert_eq!(usage.maybe_peak_bytes, None);
+    assert_eq!(usage.maybe_limit_bytes, None);
+
+    fs::write(root.join("memory.peak"), "2345\n").await.unwrap();
+    fs::write(root.join("memory.max"), "4096\n").await.unwrap();
+    let usage = read_memory_usage(root).await.unwrap();
+    assert_eq!(usage.maybe_peak_bytes, Some(2345));
+    assert_eq!(usage.maybe_limit_bytes, Some(4096));
+  }
+}

crates/service/job/video_thumbnail_job/src/job/mod.rs

@@ -1,3 +1,6 @@
 pub mod alert_on_error;
 pub mod main_loop;
 pub mod process_single_media_file;
+pub mod thumbnail_timing;
+pub mod retry_backoff;
+pub mod memory_monitor;

crates/service/job/video_thumbnail_job/src/job/process_single_media_file.rs

@@ -1,4 +1,6 @@
 use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Instant;
 
 use log::{error, info};
 use tempdir::TempDir;
@@ -11,6 +13,7 @@ use mysql_queries::queries::media_files::thumbnails::list_video_media_files_with
 use mysql_queries::queries::media_files::thumbnails::update_video_media_file_with_thumbnail::update_video_media_file_with_thumbnail;
 
 use crate::job::alert_on_error::alert_pager_and_return_err;
+use crate::job::thumbnail_timing::ThumbnailTiming;
 use crate::job_dependencies::JobDependencies;
 
 /// A downloaded video file alongside its owning temp directory.
@@ -24,6 +27,7 @@ pub struct DownloadedFile {
 pub async fn process_single_media_file(
   deps: &JobDependencies,
   media_file: &VideoMediaFileWithoutThumbnail,
+  timing: &ThumbnailTiming<'_>,
 ) -> anyhow::Result<()> {
 
   info!(
@@ -33,8 +37,11 @@ pub async fn process_single_media_file(
     media_file.created_at,
   );
 
-  let downloaded = match download_video(deps, media_file).await {
-    Ok(d) => d,
+  let stage_started_at = Instant::now();
+  let result = download_video(deps, media_file).await;
+  timing.log_stage("download", stage_started_at, result.is_ok());
+  let downloaded = match result {
+    Ok(d) => Arc::new(d),
     Err(err) => {
       error!("Failed to download video for {}: {:?}", media_file.token.as_str(), err);
       return alert_pager_and_return_err(&deps.pager, "Video download failed", err);
@@ -53,12 +60,19 @@ pub async fn process_single_media_file(
   // Generate jpg thumbnail
   let jpg_path = downloaded.temp_dir.path().join("thumbnail.jpg");
 
-  if let Err(err) = ffmpeg_video_first_frame_to_jpg_thumbnail(
-    FfmpegVideoFirstFrameToJpgThumbnailArgs {
-      input_video_path: &downloaded.file_path,
-      output_jpg_path: &jpg_path,
-    },
-  ) {
+  let stage_started_at = Instant::now();
+  // Retain the owning temp directory inside the blocking task, even if the
+  // async caller is cancelled. Await the task before releasing this job's slot.
+  let source = Arc::clone(&downloaded);
+  let destination = jpg_path.clone();
+  let result = tokio::task::spawn_blocking(move || {
+    ffmpeg_video_first_frame_to_jpg_thumbnail(FfmpegVideoFirstFrameToJpgThumbnailArgs {
+      input_video_path: &source.file_path,
+      output_jpg_path: &destination,
+    })
+  }).await.map_err(anyhow::Error::from).and_then(|result| result);
+  timing.log_stage("jpg_generation", stage_started_at, result.is_ok());
+  if let Err(err) = result {
     error!("Failed to generate JPG thumbnail for {}: {:?}", media_file.token.as_str(), err);
     return alert_pager_and_return_err(&deps.pager, "JPG thumbnail generation failed", err.into());
   }
@@ -67,7 +81,10 @@ pub async fn process_single_media_file(
 
   let jpg_object_path = format!("{video_object_path}{VIDEO_STATIC_JPG_THUMBNAIL_SUFFIX}");
 
-  if let Err(err) = deps.public_bucket_client.upload_filename(&jpg_object_path, &jpg_path).await {
+  let stage_started_at = Instant::now();
+  let result = deps.public_bucket_client.upload_filename(&jpg_object_path, &jpg_path).await;
+  timing.log_stage("jpg_upload", stage_started_at, result.is_ok());
+  if let Err(err) = result {
     error!("Failed to upload JPG thumbnail for {}: {:?}", media_file.token.as_str(), err);
     return alert_pager_and_return_err(&deps.pager, "JPG thumbnail upload failed", err);
   }
@@ -77,12 +94,17 @@ pub async fn process_single_media_file(
   // Generate gif thumbnail
   let gif_path = downloaded.temp_dir.path().join("thumbnail.gif");
 
-  if let Err(err) = ffmpeg_video_gif_preview(
-    FfmpegVideoGifPreviewArgs {
-      input_video_path: &downloaded.file_path,
-      output_gif_path: &gif_path,
-    },
-  ) {
+  let stage_started_at = Instant::now();
+  let source = Arc::clone(&downloaded);
+  let destination = gif_path.clone();
+  let result = tokio::task::spawn_blocking(move || {
+    ffmpeg_video_gif_preview(FfmpegVideoGifPreviewArgs {
+      input_video_path: &source.file_path,
+      output_gif_path: &destination,
+    })
+  }).await.map_err(anyhow::Error::from).and_then(|result| result);
+  timing.log_stage("gif_generation", stage_started_at, result.is_ok());
+  if let Err(err) = result {
     error!("Failed to generate GIF preview for {}: {:?}", media_file.token.as_str(), err);
     return alert_pager_and_return_err(&deps.pager, "GIF preview generation failed", err.into());
   }
@@ -91,7 +113,10 @@ pub async fn process_single_media_file(
 
   let gif_object_path = format!("{video_object_path}{VIDEO_ANIMATED_GIF_THUMBNAIL_SUFFIX}");
 
-  if let Err(err) = deps.public_bucket_client.upload_filename(&gif_object_path, &gif_path).await {
+  let stage_started_at = Instant::now();
+  let result = deps.public_bucket_client.upload_filename(&gif_object_path, &gif_path).await;
+  timing.log_stage("gif_upload", stage_started_at, result.is_ok());
+  if let Err(err) = result {
     error!("Failed to upload GIF preview for {}: {:?}", media_file.token.as_str(), err);
     return alert_pager_and_return_err(&deps.pager, "GIF preview upload failed", err);
   }
@@ -101,11 +126,14 @@ pub async fn process_single_media_file(
   info!("Marking thumbnail job for {:?} done", media_file.token);
 
   // Mark the media file as having a thumbnail in the database.
-  if let Err(err) = update_video_media_file_with_thumbnail(
+  let stage_started_at = Instant::now();
+  let result = update_video_media_file_with_thumbnail(
     &media_file.token,
     CURRENT_VIDEO_THUMBNAIL_VERSION,
     &deps.mysql_pool,
-  ).await {
+  ).await;
+  timing.log_stage("database_update", stage_started_at, result.is_ok());
+  if let Err(err) = result {
     error!("Failed to update thumbnail version for {}: {:?}", media_file.token.as_str(), err);
     return alert_pager_and_return_err(&deps.pager, "Thumbnail DB update failed", err.into());
   }

crates/service/job/video_thumbnail_job/src/job/retry_backoff.rs

@@ -0,0 +1,115 @@
+use std::collections::HashMap;
+use std::num::NonZeroUsize;
+use std::time::{Duration, Instant};
+
+/// Process-local failure history. Memory is bounded; restarts/eviction forget failures.
+pub struct RetryBackoff {
+  failures: HashMap<i64, Failure>,
+  capacity: NonZeroUsize,
+  initial_delay: Duration,
+  max_delay: Duration,
+}
+
+struct Failure {
+  attempts: u32,
+  last_failed_at: Instant,
+  retry_at: Instant,
+}
+
+pub struct RetrySchedule {
+  pub attempts: u32,
+  pub delay: Duration,
+}
+
+impl RetryBackoff {
+  pub fn new(capacity: NonZeroUsize, initial_delay: Duration, max_delay: Duration) -> Self {
+    Self { failures: HashMap::new(), capacity, initial_delay, max_delay }
+  }
+
+  pub fn is_ready(&self, id: i64, now: Instant) -> bool {
+    self.failures.get(&id).is_none_or(|failure| now >= failure.retry_at)
+  }
+
+  pub fn has_failed(&self, id: i64) -> bool {
+    self.failures.contains_key(&id)
+  }
+
+  pub fn record_success(&mut self, id: i64) {
+    self.failures.remove(&id);
+  }
+
+  pub fn record_failure(&mut self, id: i64, now: Instant) -> RetrySchedule {
+    if !self.failures.contains_key(&id) && self.failures.len() == self.capacity.get() {
+      // Evict the least recently failed entry. Skipping a cached failure must not
+      // refresh its position or allocate an unbounded auxiliary eviction queue.
+      if let Some(oldest_id) = self.failures.iter()
+        .min_by_key(|(_, failure)| failure.last_failed_at)
+        .map(|(id, _)| *id)
+      {
+        self.failures.remove(&oldest_id);
+      }
+    }
+
+    let attempts = self.failures.get(&id)
+      .map(|failure| failure.attempts.saturating_add(1))
+      .unwrap_or(1);
+    let multiplier = 1u32 << attempts.saturating_sub(1).min(31);
+    let delay = self.initial_delay.saturating_mul(multiplier).min(self.max_delay);
+    self.failures.insert(id, Failure { attempts, last_failed_at: now, retry_at: now + delay });
+
+    RetrySchedule { attempts, delay }
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn retries_at_the_deadline_with_exponential_delay_and_a_cap() {
+    let mut cache = cache(10);
+    let mut now = Instant::now();
+    for (index, seconds) in [30, 60, 120, 120, 120].into_iter().enumerate() {
+      let retry = cache.record_failure(1, now);
+      assert_eq!(retry.attempts, index as u32 + 1);
+      assert_eq!(retry.delay, Duration::from_secs(seconds));
+      assert!(!cache.is_ready(1, now + retry.delay - Duration::from_millis(1)));
+      now += retry.delay;
+      assert!(cache.is_ready(1, now));
+    }
+  }
+
+  #[test]
+  fn success_forgets_failures_and_resets_the_backoff() {
+    let mut cache = cache(10);
+    let now = Instant::now();
+    cache.record_failure(1, now);
+    cache.record_failure(1, now);
+    cache.record_success(1);
+    assert!(cache.is_ready(1, now));
+    assert!(!cache.has_failed(1));
+    assert_eq!(cache.record_failure(1, now).attempts, 1);
+  }
+
+  #[test]
+  fn capacity_is_bounded_and_evicts_the_least_recent_failure() {
+    let mut cache = cache(2);
+    let now = Instant::now();
+    cache.record_failure(1, now);
+    cache.record_failure(2, now + Duration::from_secs(1));
+    cache.record_failure(1, now + Duration::from_secs(2));
+    cache.record_failure(3, now + Duration::from_secs(3));
+    assert_eq!(cache.failures.len(), 2);
+    assert!(cache.has_failed(1));
+    assert!(!cache.has_failed(2));
+    assert!(cache.has_failed(3));
+    for id in 4..1000 {
+      cache.record_failure(id, now + Duration::from_secs(id as u64));
+      assert_eq!(cache.failures.len(), 2);
+    }
+  }
+
+  fn cache(capacity: usize) -> RetryBackoff {
+    RetryBackoff::new(NonZeroUsize::new(capacity).unwrap(), Duration::from_secs(30), Duration::from_secs(120))
+  }
+}

crates/service/job/video_thumbnail_job/src/job/thumbnail_timing.rs

@@ -0,0 +1,170 @@
+use std::time::Instant;
+
+use chrono::{DateTime, TimeDelta, Utc};
+use log::info;
+use serde_json::json;
+
+use mysql_queries::queries::media_files::thumbnails::list_video_media_files_without_thumbnails_for_job::VideoMediaFileWithoutThumbnail;
+
+/// Measures the complete thumbnail attempt: download through uploads and DB update.
+pub struct ThumbnailTiming<'a> {
+  media_file: &'a VideoMediaFileWithoutThumbnail,
+  clock: DatabaseClock,
+  started_at: Instant,
+}
+
+struct DatabaseClock {
+  database_read_at: DateTime<Utc>,
+  query_started_at: Instant,
+  query_finished_at: Instant,
+}
+
+struct LatencySnapshot {
+  estimated_database_at: DateTime<Utc>,
+  maybe_since_generation_completed_ms: Option<i64>,
+  since_media_created_ms: i64,
+}
+
+impl<'a> ThumbnailTiming<'a> {
+  pub fn start(
+    media_file: &'a VideoMediaFileWithoutThumbnail,
+    query_started_at: Instant,
+    query_finished_at: Instant,
+  ) -> Self {
+    let timing = Self {
+      media_file,
+      clock: DatabaseClock { database_read_at: media_file.database_read_at, query_started_at, query_finished_at },
+      started_at: Instant::now(),
+    };
+
+    timing.log_event("started", timing.started_at);
+    timing
+  }
+
+  pub fn finish(&self, success: bool) {
+    self.log_event(if success { "completed" } else { "failed" }, Instant::now());
+  }
+
+  pub fn log_stage(&self, stage: &str, started_at: Instant, success: bool) {
+    info!(
+      "thumbnail_stage {}",
+      json!({
+        "media_file_token": self.media_file.token.as_str(),
+        "stage": stage,
+        "success": success,
+        "duration_ms": started_at.elapsed().as_millis(),
+      })
+    );
+  }
+
+  fn log_event(&self, event: &str, now: Instant) {
+    let snapshot =
+      self.clock.snapshot_at(now, self.media_file.created_at, self.media_file.maybe_generation_completed_at);
+    let query_duration = self.clock.query_finished_at.duration_since(self.clock.query_started_at);
+
+    info!(
+      "thumbnail_timing {}",
+      json!({
+        "event": event,
+        "media_file_token": self.media_file.token.as_str(),
+        "media_created_at": self.media_file.created_at,
+        "generation_completed_at": self.media_file.maybe_generation_completed_at,
+        "database_read_at": self.clock.database_read_at,
+        "estimated_database_at": snapshot.estimated_database_at,
+        "since_generation_completed_ms": snapshot.maybe_since_generation_completed_ms,
+        "since_media_created_ms": snapshot.since_media_created_ms,
+        "processing_elapsed_ms": now.duration_since(self.started_at).as_millis(),
+        "page_wait_ms": self.started_at.duration_since(self.clock.query_finished_at).as_millis(),
+        "database_query_ms": query_duration.as_millis(),
+        "clock_uncertainty_ms": query_duration.as_nanos().div_ceil(2_000_000),
+        "source_timestamp_precision_ms": 1000,
+      })
+    );
+  }
+}
+
+impl DatabaseClock {
+  fn snapshot_at(
+    &self,
+    now: Instant,
+    media_created_at: DateTime<Utc>,
+    maybe_generation_completed_at: Option<DateTime<Utc>>,
+  ) -> LatencySnapshot {
+    // NOW(6) is sampled at statement start, somewhere between these two Instants.
+    // Use the midpoint as an estimate with +/- half the query duration uncertainty.
+    // This includes pool wait, execution, and network time; anchoring NOW() at the
+    // response instead would systematically omit query time from every latency.
+    // The completion/creation columns themselves have only whole-second precision.
+    let query_duration = self.query_finished_at.duration_since(self.query_started_at);
+    let midpoint = self.query_started_at + query_duration / 2;
+    let elapsed = TimeDelta::from_std(now.duration_since(midpoint))
+      .expect("thumbnail measurement duration fits in chrono::TimeDelta");
+    let estimated_database_at = self.database_read_at + elapsed;
+
+    LatencySnapshot {
+      estimated_database_at,
+      // Preserve missing and negative values instead of fabricating a zero delay.
+      maybe_since_generation_completed_ms: maybe_generation_completed_at
+        .map(|completed_at| (estimated_database_at - completed_at).num_milliseconds()),
+      since_media_created_ms: (estimated_database_at - media_created_at).num_milliseconds(),
+    }
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use std::time::Duration;
+
+  use super::*;
+
+  mod database_clock {
+    use super::*;
+
+    #[test]
+    fn includes_query_time_and_time_spent_waiting_behind_other_files() {
+      let clock = clock_with_two_second_query();
+      let created_at = clock.database_read_at - TimeDelta::seconds(12);
+      let completed_at = clock.database_read_at - TimeDelta::seconds(10);
+      let started_at = clock.query_finished_at + Duration::from_secs(60);
+
+      let start = clock.snapshot_at(started_at, created_at, Some(completed_at));
+      let end = clock.snapshot_at(started_at + Duration::from_secs(8), created_at, Some(completed_at));
+
+      assert_eq!(start.maybe_since_generation_completed_ms, Some(71_000));
+      assert_eq!(end.maybe_since_generation_completed_ms, Some(79_000));
+      assert_eq!(start.since_media_created_ms, 73_000);
+      assert_eq!(end.estimated_database_at - start.estimated_database_at, TimeDelta::seconds(8));
+    }
+
+    #[test]
+    fn missing_job_completion_does_not_become_media_creation_or_zero() {
+      let clock = clock_with_two_second_query();
+      let created_at = clock.database_read_at - TimeDelta::seconds(30);
+
+      let snapshot = clock.snapshot_at(clock.query_finished_at, created_at, None);
+
+      assert_eq!(snapshot.maybe_since_generation_completed_ms, None);
+      assert_eq!(snapshot.since_media_created_ms, 31_000);
+    }
+
+    #[test]
+    fn negative_latency_is_preserved_for_diagnosing_timestamp_anomalies() {
+      let clock = clock_with_two_second_query();
+      let completed_at = clock.database_read_at + TimeDelta::seconds(2);
+
+      let snapshot = clock.snapshot_at(clock.query_finished_at, clock.database_read_at, Some(completed_at));
+
+      assert_eq!(snapshot.maybe_since_generation_completed_ms, Some(-1_000));
+    }
+  }
+
+  fn clock_with_two_second_query() -> DatabaseClock {
+    let query_started_at = Instant::now();
+    DatabaseClock {
+      // Intentionally unrelated to the worker's wall clock.
+      database_read_at: DateTime::from_timestamp(1_000_000, 0).unwrap(),
+      query_started_at,
+      query_finished_at: query_started_at + Duration::from_secs(2),
+    }
+  }
+}

crates/service/job/video_thumbnail_job/src/job_dependencies.rs

@@ -7,6 +7,8 @@ use pager::client::pager::Pager;
 use server_environment::ServerEnvironment;
 use sqlx::MySqlPool;
 
+use crate::startup::worker_config::WorkerConfig;
+
 /// Optional sharding configuration for distributing work across parallel job instances.
 pub struct ShardInfo {
   pub number_of_shards: u8,
@@ -26,6 +28,8 @@ pub struct JobDependencies {
   /// How long to sleep between poll iterations when there is no work (milliseconds).
   pub poll_interval_millis: u64,
 
+  pub worker_config: WorkerConfig,
+
   /// Minimum delay between successive database queries within a single poll cycle (milliseconds).
   pub query_delay_millis: u64,
 

crates/service/job/video_thumbnail_job/src/main.rs

@@ -29,6 +29,7 @@ use crate::http_server::run_http_server::{launch_http_server, CreateServerArgs};
 use crate::job::main_loop::main_loop;
 use crate::job_dependencies::{JobDependencies, ShardInfo};
 use crate::startup::build_pager::build_pager;
+use crate::startup::worker_config::WorkerConfig;
 
 pub mod http_server;
 pub mod job;
@@ -98,9 +99,18 @@ async fn main() -> AnyhowResult<()> {
   // Job polling and timing configuration
   let poll_interval_millis: u64 = easyenv::get_env_num(
     "VIDEO_THUMBNAIL_POLL_INTERVAL_MILLIS",
-    30_000,
+    5_000,
   )?;
 
+  let worker_config = WorkerConfig::from_env()?;
+  info!(
+    "Thumbnail worker concurrency={} retry_cache_capacity={} retry_initial_delay_ms={} retry_max_delay_ms={}",
+    worker_config.concurrency,
+    worker_config.retry_cache_capacity,
+    worker_config.retry_initial_delay.as_millis(),
+    worker_config.retry_max_delay.as_millis(),
+  );
+
   let query_delay_millis: u64 = easyenv::get_env_num(
     "VIDEO_THUMBNAIL_QUERY_DELAY_MILLIS",
     100,
@@ -175,6 +185,7 @@ async fn main() -> AnyhowResult<()> {
     server_environment,
     job_stats,
     poll_interval_millis,
+    worker_config,
     query_delay_millis,
     query_failure_retry_delay_millis,
     custom_max_lookback_hours,

crates/service/job/video_thumbnail_job/src/startup/mod.rs

@@ -1 +1,2 @@
 pub mod build_pager;
+pub mod worker_config;

crates/service/job/video_thumbnail_job/src/startup/worker_config.rs

@@ -0,0 +1,55 @@
+use std::num::NonZeroUsize;
+use std::time::Duration;
+
+use anyhow::{ensure, Result};
+
+pub struct WorkerConfig {
+  pub concurrency: NonZeroUsize,
+  pub retry_cache_capacity: NonZeroUsize,
+  pub retry_initial_delay: Duration,
+  pub retry_max_delay: Duration,
+}
+
+impl WorkerConfig {
+  pub fn from_env() -> Result<Self> {
+    Self::new(
+      easyenv::get_env_num("VIDEO_THUMBNAIL_CONCURRENCY", 1usize)?,
+      easyenv::get_env_num("VIDEO_THUMBNAIL_RETRY_CACHE_CAPACITY", 10_000usize)?,
+      easyenv::get_env_num("VIDEO_THUMBNAIL_RETRY_INITIAL_DELAY_MILLIS", 30_000u64)?,
+      easyenv::get_env_num("VIDEO_THUMBNAIL_RETRY_MAX_DELAY_MILLIS", 1_800_000u64)?,
+    )
+  }
+
+  fn new(concurrency: usize, capacity: usize, initial_ms: u64, max_ms: u64) -> Result<Self> {
+    ensure!(concurrency > 0, "VIDEO_THUMBNAIL_CONCURRENCY must be positive");
+    ensure!(capacity > 0, "VIDEO_THUMBNAIL_RETRY_CACHE_CAPACITY must be positive");
+    ensure!(initial_ms > 0, "VIDEO_THUMBNAIL_RETRY_INITIAL_DELAY_MILLIS must be positive");
+    ensure!(max_ms >= initial_ms, "VIDEO_THUMBNAIL_RETRY_MAX_DELAY_MILLIS must be >= the initial delay");
+    ensure!(max_ms <= 86_400_000, "VIDEO_THUMBNAIL_RETRY_MAX_DELAY_MILLIS must be <= one day");
+    Ok(Self {
+      concurrency: NonZeroUsize::new(concurrency).unwrap(),
+      retry_cache_capacity: NonZeroUsize::new(capacity).unwrap(),
+      retry_initial_delay: Duration::from_millis(initial_ms),
+      retry_max_delay: Duration::from_millis(max_ms),
+    })
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn rejects_settings_that_disable_processing_or_retry_protection() {
+    for (concurrency, capacity, initial, max) in [(0, 10, 30, 60), (1, 0, 30, 60), (1, 10, 0, 60), (1, 10, 60, 30)] {
+      assert!(WorkerConfig::new(concurrency, capacity, initial, max).is_err());
+    }
+  }
+
+  #[test]
+  fn accepts_configurable_concurrency() {
+    for concurrency in 1..=4 {
+      assert_eq!(WorkerConfig::new(concurrency, 10_000, 30_000, 1_800_000).unwrap().concurrency.get(), concurrency);
+    }
+  }
+}

crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/shared_video_generation.rs

@@ -211,6 +211,8 @@ server_state.maybe_media_cdn_override_url.as_deref(),
   // ==================== PIPELINE DISPATCH ==================== //
 
   let kinovi_account = match request.model {
+    // Wan uses shared2@ (the BytePlus Ultra account) without separate Ultra model variants.
+    Some(CommonVideoModel::Wan3p0 | CommonVideoModel::Wan3p0Prime) => KinoviAccount::BytePlusUltra,
     // BytePlus Ultra
     Some(CommonVideoModel::Seedance2p0BytePlusUltra) => KinoviAccount::BytePlusUltra,
     Some(CommonVideoModel::Seedance2p0BytePlusUltraFast) => KinoviAccount::BytePlusUltra,

frontend/apps/artcraft-webapp/src/components/sidebar/library-folders-nav.tsx

@@ -2,7 +2,14 @@ import { SidebarActiveIndicator } from "./sidebar-active-indicator";
 import { useMemo, useState } from "react";
 import { Link, useNavigate } from "react-router-dom";
 import { AnimatePresence, motion } from "framer-motion";
-import { ChevronRightIcon, FolderIcon, FolderOpenIcon, Grid3x3Icon, PlusIcon, StarIcon } from "lucide-react";
+import {
+  ChevronRightIcon,
+  FolderIcon,
+  FolderOpenIcon,
+  Grid3x3Icon,
+  PlusIcon,
+  StarIcon,
+} from "lucide-react";
 import { compareFolders } from "@storyteller/ui-gallery-modal";
 import { EASE_EMPHASIS } from "../../lib/motion";
 import {
@@ -15,6 +22,7 @@ import {
 } from "../ui/sidebar";
 import { useLibraryFoldersStore } from "../../pages/library/library-folders-store";
 import { LibraryTagsNav } from "./library-tags-nav";
+import { useFolderDrag } from "../../pages/library/use-folder-drag";
 
 /**
  * The "Library" sidebar section: Unsorted + Folders entries, plus the user's
@@ -29,9 +37,9 @@ export function LibraryFoldersNav({
   onNavClick: () => void;
 }) {
   const navigate = useNavigate();
+  const folderDrag = useFolderDrag();
   const onFolders =
-    pathname === "/library/folders" ||
-    pathname.startsWith("/library/folder_");
+    pathname === "/library/folders" || pathname.startsWith("/library/folder_");
   const onFolderless = pathname === "/library/folderless";
   const onTags =
     pathname === "/library/tags" || pathname.startsWith("/library/tag_");
@@ -44,7 +52,9 @@ export function LibraryFoldersNav({
 
   const folders = useLibraryFoldersStore((s) => s.folders);
   const activeFolderId = useLibraryFoldersStore((s) => s.activeFolderId);
-  const openNewFolderModal = useLibraryFoldersStore((s) => s.openNewFolderModal);
+  const openNewFolderModal = useLibraryFoldersStore(
+    (s) => s.openNewFolderModal,
+  );
   const setContextMenu = useLibraryFoldersStore((s) => s.setContextMenu);
 
   // Folders are the primary organization tool — default expanded.
@@ -55,18 +65,57 @@ export function LibraryFoldersNav({
     [folders],
   );
 
-  // Highlight whichever root branch the active folder lives in.
-  const activeRootId = useMemo(() => {
-    if (!activeFolderId) return null;
+  // Reveal the active folder's ancestors and its immediate children.
+  const activeBranch = useMemo(() => {
     const byId = new Map(folders.map((f) => [f.id, f]));
     const seen = new Set<string>();
-    let cursor = byId.get(activeFolderId);
-    while (cursor && cursor.parentId && !seen.has(cursor.id)) {
+    let cursor = activeFolderId ? byId.get(activeFolderId) : undefined;
+    while (cursor && !seen.has(cursor.id)) {
       seen.add(cursor.id);
-      cursor = byId.get(cursor.parentId);
+      cursor = cursor.parentId ? byId.get(cursor.parentId) : undefined;
     }
-    return cursor?.id ?? null;
+    return seen;
   }, [folders, activeFolderId]);
+  const [branchOverrides, setBranchOverrides] = useState<{
+    activeId: string | null;
+    values: Record<string, boolean>;
+  }>({ activeId: null, values: {} });
+  const visibleFolders = useMemo(() => {
+    const children = new Map<string | null, typeof folders>();
+    for (const folder of folders) {
+      const siblings = children.get(folder.parentId) ?? [];
+      siblings.push(folder);
+      children.set(folder.parentId, siblings);
+    }
+    for (const siblings of children.values()) siblings.sort(compareFolders);
+    const rows: {
+      folder: (typeof folders)[number];
+      depth: number;
+      hasChildren: boolean;
+      open: boolean;
+    }[] = [];
+    const seen = new Set<string>();
+    const visit = (parentId: string | null, depth: number) => {
+      for (const folder of children.get(parentId) ?? []) {
+        if (seen.has(folder.id)) continue;
+        seen.add(folder.id);
+        const override =
+          branchOverrides.activeId === activeFolderId
+            ? branchOverrides.values[folder.id]
+            : undefined;
+        const open = override ?? activeBranch.has(folder.id);
+        rows.push({
+          folder,
+          depth,
+          hasChildren: children.has(folder.id),
+          open,
+        });
+        if (open) visit(folder.id, depth + 1);
+      }
+    };
+    visit(null, 0);
+    return rows;
+  }, [folders, activeBranch, activeFolderId, branchOverrides]);
 
   const goToFolder = (id: string) => {
     navigate(`/library/${id}`);
@@ -74,7 +123,7 @@ export function LibraryFoldersNav({
   };
 
   return (
-    <SidebarGroup>
+    <SidebarGroup {...folderDrag}>
       <div className="flex items-center justify-between">
         <SidebarGroupLabel>Assets</SidebarGroupLabel>
         {inLibraryArea && (
@@ -84,7 +133,7 @@ export function LibraryFoldersNav({
             aria-label="New folder"
             className="mr-1 flex h-5 w-5 items-center justify-center text-sidebar-foreground/50 hover:bg-sidebar-accent hover:text-sidebar-foreground transition-colors group-data-[collapsible=icon]:hidden"
           >
-            <PlusIcon  className="text-xs" />
+            <PlusIcon className="text-xs" />
           </button>
         )}
       </div>
@@ -119,7 +168,12 @@ export function LibraryFoldersNav({
               isActive={onFolders && !activeFolderId}
               tooltip="Folders"
             >
-              <Link to="/library/folders" onClick={onNavClick}>
+              <Link
+                to="/library/folders"
+                onClick={onNavClick}
+                data-folder-root
+                className="[&.folder-drag-over]:bg-primary/20"
+              >
                 <FolderIcon />
                 <span>Folders</span>
               </Link>
@@ -132,8 +186,8 @@ export function LibraryFoldersNav({
                 className="absolute right-1 top-1/2 -translate-y-1/2 flex h-5 w-5 items-center justify-center text-sidebar-foreground/50 hover:bg-sidebar-accent hover:text-sidebar-foreground transition-colors group-data-[collapsible=icon]:hidden"
               >
                 <ChevronRightIcon
-                  
-                  className={`text-[10px] transition-transform duration-200 ${expanded ? "rotate-90" : ""}`} />
+                  className={`text-[10px] transition-transform duration-200 ${expanded ? "rotate-90" : ""}`}
+                />
               </button>
             )}
           </SidebarMenuItem>
@@ -149,41 +203,69 @@ export function LibraryFoldersNav({
                 className="list-none overflow-hidden group-data-[collapsible=icon]:hidden"
               >
                 <ul className="flex w-full min-w-0 flex-col gap-0.5">
-                  {rootFolders.map((folder) => (
-                    <SidebarMenuItem key={folder.id}>
-                      {onFolders && activeRootId === folder.id && <SidebarActiveIndicator />}
-                      <SidebarMenuButton
-                        isActive={onFolders && activeRootId === folder.id}
-                        tooltip={folder.name}
-                        data-folder-id={folder.id}
-                        onClick={() => goToFolder(folder.id)}
-                        onContextMenu={(e) => {
-                          e.preventDefault();
-                          setContextMenu({
-                            folderId: folder.id,
-                            x: e.clientX,
-                            y: e.clientY,
-                          });
-                        }}
-                        className="pl-5 [&.folder-drag-over]:bg-primary/20 [&.folder-drag-over]:text-sidebar-foreground"
-                      >
-                        <FolderIcon
-                          
-                          className={folder.colorCode ? "" : "text-primary"}
-                          style={
-                            folder.colorCode
-                              ? { color: folder.colorCode }
-                              : undefined
-                          } />
-                        <span className="truncate">{folder.name}</span>
-                        {folder.hasStar && (
-                          <StarIcon
-                            
-                            className="ml-auto text-[10px] text-amber-400" />
+                  {visibleFolders.map(
+                    ({ folder, depth, hasChildren, open }) => (
+                      <SidebarMenuItem key={folder.id}>
+                        {onFolders && activeFolderId === folder.id && (
+                          <SidebarActiveIndicator />
                         )}
-                      </SidebarMenuButton>
-                    </SidebarMenuItem>
-                  ))}
+                        <SidebarMenuButton
+                          isActive={onFolders && activeFolderId === folder.id}
+                          tooltip={folder.name}
+                          data-folder-id={folder.id}
+                          draggable
+                          style={{ paddingLeft: 28 + depth * 16 }}
+                          onClick={() => goToFolder(folder.id)}
+                          onContextMenu={(e) => {
+                            e.preventDefault();
+                            setContextMenu({
+                              folderId: folder.id,
+                              x: e.clientX,
+                              y: e.clientY,
+                            });
+                          }}
+                          className="pl-5 [&.folder-drag-over]:bg-primary/20 [&.folder-drag-over]:text-sidebar-foreground"
+                        >
+                          <FolderIcon
+                            className={folder.colorCode ? "" : "text-primary"}
+                            style={
+                              folder.colorCode
+                                ? { color: folder.colorCode }
+                                : undefined
+                            }
+                          />
+                          <span className="truncate">{folder.name}</span>
+                          {folder.hasStar && (
+                            <StarIcon className="ml-auto text-[10px] text-amber-400" />
+                          )}
+                        </SidebarMenuButton>
+                        {hasChildren && (
+                          <button
+                            type="button"
+                            aria-label={`${open ? "Collapse" : "Expand"} ${folder.name}`}
+                            aria-expanded={open}
+                            className="absolute top-1/2 -translate-y-1/2 flex h-5 w-5 items-center justify-center rounded hover:bg-sidebar-accent"
+                            style={{ left: 4 + depth * 16 }}
+                            onClick={() =>
+                              setBranchOverrides((previous) => ({
+                                activeId: activeFolderId,
+                                values: {
+                                  ...(previous.activeId === activeFolderId
+                                    ? previous.values
+                                    : {}),
+                                  [folder.id]: !open,
+                                },
+                              }))
+                            }
+                          >
+                            <ChevronRightIcon
+                              className={`h-3 w-3 transition-transform ${open ? "rotate-90" : ""}`}
+                            />
+                          </button>
+                        )}
+                      </SidebarMenuItem>
+                    ),
+                  )}
                 </ul>
               </motion.li>
             )}

frontend/apps/artcraft-webapp/src/pages/library/folder-move.spec.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest";
+import { getFolderMoveDestinations, getFolderPath } from "./folder-move";
+
+const folders = [
+  { id: "a", name: "Projects", parentId: null },
+  { id: "b", name: "Images", parentId: "a" },
+  { id: "c", name: "Drafts", parentId: "b" },
+  { id: "d", name: "Images", parentId: null },
+];
+
+describe("folder move destinations", () => {
+  it("excludes the complete source subtree but allows ancestors and other branches", () => {
+    expect(getFolderMoveDestinations(folders, "b").map((f) => f.id)).toEqual([
+      "a",
+      "d",
+    ]);
+    expect(getFolderMoveDestinations(folders, "a").map((f) => f.id)).toEqual([
+      "d",
+    ]);
+  });
+
+  it("distinguishes duplicate names using their full paths", () => {
+    const byId = new Map(folders.map((f) => [f.id, f]));
+    expect(getFolderPath(folders[1], byId)).toBe("Projects / Images");
+    expect(getFolderPath(folders[3], byId)).toBe("Images");
+  });
+
+  it("terminates even if existing folder data contains a cycle", () => {
+    const cyclic = [{ ...folders[0], parentId: "c" }, ...folders.slice(1)];
+    expect(getFolderMoveDestinations(cyclic, "a").map((f) => f.id)).toEqual([
+      "d",
+    ]);
+  });
+});

frontend/apps/artcraft-webapp/src/pages/library/folder-move.ts

@@ -0,0 +1,44 @@
+interface FolderNode {
+  id: string;
+  name: string;
+  parentId: string | null;
+}
+
+/** Exclude the source subtree so a move can never create a parent cycle. */
+export function getFolderMoveDestinations<T extends FolderNode>(
+  folders: T[],
+  sourceId: string,
+): T[] {
+  const children = new Map<string, string[]>();
+  for (const folder of folders) {
+    if (!folder.parentId) continue;
+    const siblings = children.get(folder.parentId) ?? [];
+    siblings.push(folder.id);
+    children.set(folder.parentId, siblings);
+  }
+  const excluded = new Set<string>();
+  const pending = [sourceId];
+  while (pending.length) {
+    const id = pending.pop()!;
+    if (excluded.has(id)) continue;
+    excluded.add(id);
+    pending.push(...(children.get(id) ?? []));
+  }
+  return folders.filter((folder) => !excluded.has(folder.id));
+}
+
+/** Full paths distinguish identically named folders in different branches. */
+export function getFolderPath(
+  folder: FolderNode,
+  byId: Map<string, FolderNode>,
+): string {
+  const names: string[] = [];
+  const seen = new Set<string>();
+  let current: FolderNode | undefined = folder;
+  while (current && !seen.has(current.id)) {
+    seen.add(current.id);
+    names.unshift(current.name);
+    current = current.parentId ? byId.get(current.parentId) : undefined;
+  }
+  return names.join(" / ");
+}

frontend/apps/artcraft-webapp/src/pages/library/library-folders-store.spec.ts

@@ -0,0 +1,85 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { useLibraryFoldersStore } from "./library-folders-store";
+
+const { add, remove, error } = vi.hoisted(() => ({
+  add: vi.fn(),
+  remove: vi.fn(),
+  error: vi.fn(),
+}));
+vi.mock("@storyteller/api", () => ({
+  FoldersApi: class {
+    AddSubfolders = add;
+    RemoveSubfolders = remove;
+  },
+  MediaFilesApi: class {},
+}));
+vi.mock("@storyteller/ui-gallery-modal", () => ({}));
+vi.mock("./library-media-map", () => ({ errMsg: String }));
+vi.mock("./library-tags-store", () => ({}));
+vi.mock("../../components/toast/toast", () => ({
+  toast: { error, success: vi.fn() },
+}));
+
+describe("moveFolder", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    useLibraryFoldersStore.setState({
+      folders: [
+        { id: "source", name: "Source", parentId: null },
+        { id: "child", name: "Child", parentId: "source" },
+        { id: "target", name: "Target", parentId: null },
+      ],
+      folderMediaItems: { source: [], child: [] },
+    });
+  });
+
+  it("reparents only the source and preserves descendants and cached media", async () => {
+    add.mockResolvedValue({ success: true, data: ["source"] });
+    const media = useLibraryFoldersStore.getState().folderMediaItems;
+    expect(
+      await useLibraryFoldersStore.getState().moveFolder("source", "target"),
+    ).toBe(true);
+    expect(add).toHaveBeenCalledWith({
+      folderToken: "target",
+      subfolderTokens: ["source"],
+    });
+    const state = useLibraryFoldersStore.getState();
+    expect(state.folders[0].parentId).toBe("target");
+    expect(state.folders[1].parentId).toBe("source");
+    expect(state.folderMediaItems).toBe(media);
+  });
+
+  it("moves a nested folder to the top level", async () => {
+    remove.mockResolvedValue({ success: true, data: 1 });
+    expect(
+      await useLibraryFoldersStore.getState().moveFolder("child", null),
+    ).toBe(true);
+    expect(remove).toHaveBeenCalledWith({
+      folderToken: "source",
+      subfolderTokens: ["child"],
+    });
+    expect(useLibraryFoldersStore.getState().folders[1].parentId).toBeNull();
+  });
+
+  it("rejects moves into the source subtree before calling the API", async () => {
+    for (const target of ["source", "child"]) {
+      expect(
+        await useLibraryFoldersStore.getState().moveFolder("source", target),
+      ).toBe(false);
+    }
+    expect(add).not.toHaveBeenCalled();
+  });
+
+  it.each([{ success: false }, { success: true, data: [] }])(
+    "preserves the tree when the API does not accept the move: %j",
+    async (response) => {
+      add.mockResolvedValue(response);
+      const folders = useLibraryFoldersStore.getState().folders;
+      expect(
+        await useLibraryFoldersStore.getState().moveFolder("source", "target"),
+      ).toBe(false);
+      expect(useLibraryFoldersStore.getState().folders).toBe(folders);
+      expect(error).toHaveBeenCalled();
+    },
+  );
+});

frontend/apps/artcraft-webapp/src/pages/library/library-folders-store.ts

@@ -9,6 +9,7 @@ import {
 import { toast } from "../../components/toast/toast";
 import { errMsg, mapLeanListItemToGalleryItem } from "./library-media-map";
 import { useLibraryTagsStore } from "./library-tags-store";
+import { getFolderMoveDestinations } from "./folder-move";
 
 // ── Bulk selection store ────────────────────────────────────────────────────
 // Kept in its own module store (not page state) so each gallery tile can
@@ -92,6 +93,7 @@ interface LibraryFoldersState {
     parentId: string | null,
   ) => Promise<UiFolder | null>;
   renameFolder: (folderId: string, name: string) => Promise<void>;
+  moveFolder: (folderId: string, parentId: string | null) => Promise<boolean>;
   setFolderStar: (folderId: string, hasStar: boolean) => Promise<void>;
   setFolderColor: (folderId: string, colorCode: string | null) => Promise<void>;
   deleteFolder: (folderId: string) => Promise<void>;
@@ -209,6 +211,51 @@ export const useLibraryFoldersStore = create<LibraryFoldersState>(
       }
     },
 
+    moveFolder: async (folderId, parentId) => {
+      const folders = get().folders;
+      const folder = folders.find((f) => f.id === folderId);
+      if (!folder) return false;
+      if (folder.parentId === parentId) return true;
+      if (
+        parentId &&
+        !getFolderMoveDestinations(folders, folderId).some((f) => f.id === parentId)
+      ) {
+        toast.error(
+          "Choose a destination outside this folder and its subfolders.",
+        );
+        return false;
+      }
+      try {
+        const result = parentId
+          ? await foldersApi.AddSubfolders({
+              folderToken: parentId,
+              subfolderTokens: [folderId],
+            })
+          : await foldersApi.RemoveSubfolders({
+              folderToken: folder.parentId!,
+              subfolderTokens: [folderId],
+            });
+        const accepted = parentId
+          ? Array.isArray(result.data) && result.data.includes(folderId)
+          : typeof result.data === "number" && result.data > 0;
+        if (!result.success || !accepted) {
+          toast.error(result.errorMessage || "Could not move folder. Please try again.");
+          return false;
+        }
+        // Only the moved node changes; its media and descendant links stay intact.
+        set((s) => ({
+          folders: s.folders.map((f) =>
+            f.id === folderId ? { ...f, parentId } : f,
+          ),
+        }));
+        toast.success("Folder moved");
+        return true;
+      } catch (err) {
+        toast.error(`Failed to move folder: ${errMsg(err)}`);
+        return false;
+      }
+    },
+
     setActiveFolder: (id) => {
       set({ activeFolderId: id, contextMenu: null });
       if (id) get().loadFolderMedia(id, true);

frontend/apps/artcraft-webapp/src/pages/library/library.tsx

@@ -77,6 +77,8 @@ import {
   type UiFolder,
 } from "./library-folders-store";
 import { mapRawToGalleryItem } from "./library-media-map";
+import { MoveFolderDialog } from "./move-folder-dialog";
+import { useFolderDrag } from "./use-folder-drag";
 import {
   compareTagsByUseCount,
   useLibraryTagsStore,
@@ -177,6 +179,7 @@ const GRID_CLASS =
 // ── Component ──────────────────────────────────────────────────────────────
 
 export default function Library() {
+  const folderDrag = useFolderDrag();
   // `:slug` is either a media-class filter (images/videos/meshes), a folder
   // token (prefixed `folder_`), the static `tags` tab, or a tag token
   // (prefixed `tag_`). `/library/folders` (static) has no slug.
@@ -250,6 +253,8 @@ export default function Library() {
   const newFolderModal = useLibraryFoldersStore((s) => s.newFolderModal);
   const renameTarget = useLibraryFoldersStore((s) => s.renameTarget);
   const contextMenu = useLibraryFoldersStore((s) => s.contextMenu);
+  const [moveTarget, setMoveTarget] = useState<string | null>(null);
+  const movingFolder = folders.find((f) => f.id === moveTarget);
   const loadFolders = useLibraryFoldersStore((s) => s.loadFolders);
   const setActiveFolder = useLibraryFoldersStore((s) => s.setActiveFolder);
   const createFolder = useLibraryFoldersStore((s) => s.createFolder);
@@ -617,11 +622,13 @@ export default function Library() {
     (e: React.PointerEvent<HTMLDivElement>) => {
       if (e.button !== 0 || lightboxOpen) return;
       const target = e.target as HTMLElement;
+      // React portal events bubble here even though dialogs are outside the grid.
+      if (!e.currentTarget.contains(target)) return;
       // Start only on blank background — never from tiles, folder cards,
       // controls, or opted-out chrome (header / bulk bar).
       if (
         target.closest(
-          "[data-media-id], [data-folder-id], button, a, input, [data-no-marquee]",
+          "[data-media-id], [data-folder-id], button, a, input, select, textarea, [role='dialog'], [data-no-marquee]",
         )
       ) {
         return;
@@ -1106,6 +1113,7 @@ export default function Library() {
       // hijack the drag. Form fields opt back in so dialog inputs stay editable.
       className="relative min-h-full w-full shrink-0 select-none [&_input]:select-text [&_textarea]:select-text bg-ui-background pb-8 px-3 sm:px-4 md:px-8 lg:px-12"
       onPointerDown={handleMarqueePointerDown}
+      {...folderDrag}
     >
       <div className="mx-auto max-w-[1600px]">
         {/* Header — sticky below navbar */}
@@ -1446,6 +1454,7 @@ export default function Library() {
               <div className={GRID_CLASS}>
                 {currentSubfolders.map((folder) => (
                   <GalleryFolderChip
+                    draggable
                     key={folder.id}
                     folder={folder}
                     childCount={subfolderCount(folder.id)}
@@ -1728,6 +1737,14 @@ export default function Library() {
         onClose={closeNewFolderModal}
       />
 
+      {/* Move folder dialog */}
+      {movingFolder && (
+        <MoveFolderDialog
+          key={movingFolder.id}
+          folder={movingFolder}
+          onClose={() => setMoveTarget(null)}
+        />
+      )}
       {/* Rename dialog */}
       <FolderNameDialog
         isOpen={!!renameTarget}
@@ -1816,6 +1833,17 @@ export default function Library() {
                 <PencilIcon className="w-4" />
                 <span>Rename</span>
               </button>
+              <button
+                type="button"
+                className="flex w-full items-center gap-2 px-2 py-2 hover:bg-ui-controls/60 text-sm text-base-fg"
+                onClick={() => {
+                  setMoveTarget(contextMenu.folderId);
+                  setContextMenu(null);
+                }}
+              >
+                <FolderOpenIcon className="w-4" />
+                <span>Move folder…</span>
+              </button>
               <button
                 type="button"
                 className="flex w-full items-center gap-2 px-2 py-2 hover:bg-ui-controls/60 text-sm text-red"

frontend/apps/artcraft-webapp/src/pages/library/move-folder-dialog.tsx

@@ -0,0 +1,101 @@
+import { useMemo, useRef, useState } from "react";
+import { Modal } from "@storyteller/ui-modal";
+import { Button } from "@storyteller/ui-button";
+import { ChevronDownIcon } from "lucide-react";
+import { getFolderMoveDestinations, getFolderPath } from "./folder-move";
+import { useLibraryFoldersStore, type UiFolder } from "./library-folders-store";
+
+export function MoveFolderDialog({
+  folder,
+  onClose,
+}: {
+  folder: UiFolder;
+  onClose: () => void;
+}) {
+  const folders = useLibraryFoldersStore((s) => s.folders);
+  const moveFolder = useLibraryFoldersStore((s) => s.moveFolder);
+  const [destination, setDestination] = useState(folder.parentId ?? "");
+  const [moving, setMoving] = useState(false);
+  const inFlight = useRef(false);
+  const destinations = useMemo(() => {
+    const byId = new Map(folders.map((f) => [f.id, f]));
+    return getFolderMoveDestinations(folders, folder.id)
+      .map((f) => ({ id: f.id, path: getFolderPath(f, byId) }))
+      .sort((a, b) => a.path.localeCompare(b.path));
+  }, [folders, folder.id]);
+  const validDestination =
+    destination === "" || destinations.some((f) => f.id === destination);
+
+  const submit = async () => {
+    if (
+      inFlight.current ||
+      !validDestination ||
+      destination === (folder.parentId ?? "")
+    )
+      return;
+    inFlight.current = true;
+    setMoving(true);
+    try {
+      if (await moveFolder(folder.id, destination || null)) onClose();
+    } finally {
+      inFlight.current = false;
+      setMoving(false);
+    }
+  };
+
+  return (
+    <Modal
+      isOpen
+      onClose={() => {
+        if (!inFlight.current) onClose();
+      }}
+      title="Move folder"
+      accessibleTitle="Move folder"
+      className="w-96 max-w-[90vw] [&_h2]:font-sans [&_h2]:text-sm [&_h2]:font-semibold [&_h2]:tracking-normal [&_h2]:[font-stretch:normal]"
+    >
+      <div className="space-y-3">
+        <p className="text-sm text-base-fg/70">
+          Move “{folder.name}” with all its items and subfolders.
+        </p>
+        <label className="block space-y-1 text-sm">
+          <span>Destination</span>
+          <div className="relative">
+            <select
+              value={destination}
+              onChange={(e) => setDestination(e.target.value)}
+              disabled={moving}
+              autoFocus
+              className="w-full appearance-none rounded-[3px] border border-ui-panel-border bg-ui-panel py-2 pl-3 pr-10 text-sm text-base-fg outline-none focus:ring-1 focus:ring-primary/50"
+            >
+              <option value="">Folders (top level)</option>
+              {destinations.map((f) => (
+                <option key={f.id} value={f.id}>
+                  {f.path}
+                </option>
+              ))}
+            </select>
+            <ChevronDownIcon
+              aria-hidden="true"
+              className="pointer-events-none absolute right-3 top-1/2 h-4 w-4 -translate-y-1/2 text-base-fg/70"
+            />
+          </div>
+        </label>
+        <div className="flex justify-end gap-2">
+          <Button variant="action" onClick={onClose} disabled={moving}>
+            Cancel
+          </Button>
+          <Button
+            onClick={submit}
+            disabled={
+              moving ||
+              !validDestination ||
+              destination === (folder.parentId ?? "")
+            }
+          >
+            {moving ? "Moving…" : "Move"}
+          </Button>
+        </div>
+      </div>
+    </Modal>
+  );
+}

frontend/apps/artcraft-webapp/src/pages/library/use-folder-drag.spec.ts

@@ -0,0 +1,76 @@
+import { renderHook } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { DragEvent } from "react";
+import { useFolderDrag } from "./use-folder-drag";
+
+const { moveFolder } = vi.hoisted(() => ({ moveFolder: vi.fn() }));
+vi.mock("./library-folders-store", () => ({
+  useLibraryFoldersStore: {
+    getState: () => ({
+      folders: [
+        { id: "source", name: "Source", parentId: "parent" },
+        { id: "child", name: "Child", parentId: "source" },
+        { id: "target", name: "Target", parentId: null },
+        { id: "parent", name: "Parent", parentId: null },
+      ],
+      moveFolder,
+    }),
+  },
+}));
+
+afterEach(() => {
+  document.body.innerHTML = "";
+  vi.clearAllMocks();
+});
+
+describe("folder dragging across grid and sidebar", () => {
+  it.each(["target", null])("moves to destination %s", (destination) => {
+    const grid = renderHook(useFolderDrag).result.current;
+    const sidebar = renderHook(useFolderDrag).result.current;
+    grid.onDragStart(dragEvent("source"));
+    const drop = dragEvent(destination);
+    sidebar.onDragOver(drop);
+    expect(drop.preventDefault).toHaveBeenCalled();
+    expect((drop.target as HTMLElement).classList.contains("folder-drag-over")).toBe(true);
+    sidebar.onDrop(drop);
+    expect(moveFolder).toHaveBeenCalledWith("source", destination);
+    expect(document.querySelector(".folder-drag-over")).toBeNull();
+    grid.onDragEnd();
+  });
+
+  it.each(["source", "child", "parent"])("rejects invalid or unchanged destination %s", (destination) => {
+    const drag = renderHook(useFolderDrag).result.current;
+    drag.onDragStart(dragEvent("source"));
+    const drop = dragEvent(destination);
+    drag.onDragOver(drop);
+    drag.onDrop(drop);
+    expect(drop.preventDefault).not.toHaveBeenCalled();
+    expect(moveFolder).not.toHaveBeenCalled();
+    drag.onDragEnd();
+  });
+
+  it("does not accept drops after cancellation", () => {
+    const drag = renderHook(useFolderDrag).result.current;
+    drag.onDragStart(dragEvent("source"));
+    drag.onDragEnd();
+    drag.onDrop(dragEvent("target"));
+    expect(moveFolder).not.toHaveBeenCalled();
+  });
+});
+
+function dragEvent(id: string | null): DragEvent<HTMLElement> {
+  const container = document.createElement("div");
+  const target = document.createElement("button");
+  target.draggable = true;
+  if (id) target.dataset.folderId = id;
+  else target.dataset.folderRoot = "";
+  container.append(target);
+  document.body.append(container);
+  return {
+    target,
+    currentTarget: container,
+    dataTransfer: { setData: vi.fn() },
+    preventDefault: vi.fn(),
+    stopPropagation: vi.fn(),
+  } as unknown as DragEvent<HTMLElement>;
+}

frontend/apps/artcraft-webapp/src/pages/library/use-folder-drag.ts

@@ -0,0 +1,79 @@
+import { useMemo } from "react";
+import type { DragEvent } from "react";
+import { getFolderMoveDestinations } from "./folder-move";
+import { useLibraryFoldersStore } from "./library-folders-store";
+
+const FOLDER_MIME = "application/x-artcraft-folder";
+let draggedFolderId: string | null = null;
+
+/** Shared native drag handlers for the library grid and sidebar. */
+export function useFolderDrag() {
+  return useMemo(() => {
+    const clearHighlight = () => {
+      document.querySelectorAll(".folder-drag-over").forEach((el) => {
+        el.classList.remove("folder-drag-over");
+      });
+    };
+    const targetAt = (event: DragEvent<HTMLElement>) => {
+      const target = (event.target as HTMLElement).closest<HTMLElement>(
+        "[data-folder-id], [data-folder-root]",
+      );
+      if (!target || !event.currentTarget.contains(target)) return null;
+      return target;
+    };
+    const canDrop = (target: HTMLElement | null) => {
+      if (!draggedFolderId || !target) return false;
+      const { folders } = useLibraryFoldersStore.getState();
+      const source = folders.find((f) => f.id === draggedFolderId);
+      const parentId = target.dataset.folderId ?? null;
+      return (
+        !!source &&
+        source.parentId !== parentId &&
+        (!parentId ||
+          getFolderMoveDestinations(folders, source.id).some(
+            (f) => f.id === parentId,
+          ))
+      );
+    };
+    return {
+      onDragStart(event: DragEvent<HTMLElement>) {
+        const source = targetAt(event);
+        if (!source?.dataset.folderId || !source.draggable) return;
+        draggedFolderId = source.dataset.folderId;
+        event.dataTransfer.setData(FOLDER_MIME, draggedFolderId);
+        event.dataTransfer.effectAllowed = "move";
+        event.stopPropagation();
+      },
+      onDragOver(event: DragEvent<HTMLElement>) {
+        if (!draggedFolderId) return;
+        clearHighlight();
+        const target = targetAt(event);
+        if (canDrop(target)) {
+          event.preventDefault();
+          event.dataTransfer.dropEffect = "move";
+          target!.classList.add("folder-drag-over");
+        }
+      },
+      onDragLeave(event: DragEvent<HTMLElement>) {
+        if (!event.currentTarget.contains(event.relatedTarget as Node | null))
+          clearHighlight();
+      },
+      onDrop(event: DragEvent<HTMLElement>) {
+        const target = targetAt(event);
+        if (!canDrop(target)) return;
+        event.preventDefault();
+        event.stopPropagation();
+        const sourceId = draggedFolderId!;
+        draggedFolderId = null;
+        clearHighlight();
+        void useLibraryFoldersStore
+          .getState()
+          .moveFolder(sourceId, target!.dataset.folderId ?? null);
+      },
+      onDragEnd() {
+        draggedFolderId = null;
+        clearHighlight();
+      },
+    };
+  }, []);
+}

frontend/libs/components/gallery-modal/src/lib/GalleryFolderChip.tsx

@@ -7,6 +7,7 @@ interface GalleryFolderChipProps {
   folder: GalleryFolder;
   /** Direct subfolder count, shown as a subtitle. */
   childCount: number;
+  draggable?: boolean;
   onOpen: (folderId: string) => void;
   /**
    * Opens the shared folder context menu (rename / delete / color / star /
@@ -41,6 +42,7 @@ const ChipImg = ({ src, className }: { src: string; className: string }) => {
 export const GalleryFolderChip: React.FC<GalleryFolderChipProps> = ({
   folder,
   childCount,
+  draggable = false,
   onOpen,
   onContextMenu,
 }) => {
@@ -60,6 +62,7 @@ export const GalleryFolderChip: React.FC<GalleryFolderChipProps> = ({
     <button
       type="button"
       data-folder-id={folder.id}
+      draggable={draggable}
       onClick={() => onOpen(folder.id)}
       onContextMenu={(e) => {
         e.preventDefault();