From c6f3e042524de5aaef8442402b282181e1b91abe Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 31 Mar 2026 03:19:24 -0400 Subject: [PATCH] main loop errors --- .../src/job/alert_on_error.rs | 20 ++++++++ .../video_thumbnail_job/src/job/main_loop.rs | 2 + .../job/video_thumbnail_job/src/job/mod.rs | 1 + .../src/job/process_single_media_file.rs | 50 ++++++++++++------- 4 files changed, 56 insertions(+), 17 deletions(-) create mode 100644 crates/service/job/video_thumbnail_job/src/job/alert_on_error.rs diff --git a/crates/service/job/video_thumbnail_job/src/job/alert_on_error.rs b/crates/service/job/video_thumbnail_job/src/job/alert_on_error.rs new file mode 100644 index 0000000000..2409136895 --- /dev/null +++ b/crates/service/job/video_thumbnail_job/src/job/alert_on_error.rs @@ -0,0 +1,20 @@ +use log::error; +use pager::client::pager::Pager; +use pager::notification::notification_details_builder::NotificationDetailsBuilder; +use pager::notification::notification_urgency::NotificationUrgency; + +/// Enqueue a pager alert for the error, then return it as `Err`. +pub fn alert_pager_and_return_err( + pager: &Pager, + err: anyhow::Error, +) -> anyhow::Result { + let notification = NotificationDetailsBuilder::from_error(&err) + .set_urgency(Some(NotificationUrgency::Medium)) + .build(); + + if let Err(pager_err) = pager.enqueue_page(notification) { + error!("Failed to enqueue pager alert: {:?}", pager_err); + } + + Err(err) +} diff --git a/crates/service/job/video_thumbnail_job/src/job/main_loop.rs b/crates/service/job/video_thumbnail_job/src/job/main_loop.rs index b6b09f7b2b..716b2d2859 100644 --- a/crates/service/job/video_thumbnail_job/src/job/main_loop.rs +++ b/crates/service/job/video_thumbnail_job/src/job/main_loop.rs @@ -7,6 +7,7 @@ use mysql_queries::queries::media_files::thumbnails::list_video_media_files_with ListVideoMediaFilesWithoutThumbnailsArgs, }; +use crate::job::alert_on_error::alert_pager_and_return_err; use crate::job_dependencies::JobDependencies; use crate::job::process_single_media_file::process_single_media_file; @@ -21,6 +22,7 @@ pub async fn main_loop(deps: JobDependencies) { } Err(err) => { error!("Error in video thumbnail batch cycle: {:?}", err); + let _ = alert_pager_and_return_err::<()>(&deps.pager, err); let _ = deps.job_stats.increment_failure_count(); // Wait before retrying after a failure. diff --git a/crates/service/job/video_thumbnail_job/src/job/mod.rs b/crates/service/job/video_thumbnail_job/src/job/mod.rs index 4760a9a3b3..8018e2688c 100644 --- a/crates/service/job/video_thumbnail_job/src/job/mod.rs +++ b/crates/service/job/video_thumbnail_job/src/job/mod.rs @@ -1,2 +1,3 @@ +pub mod alert_on_error; pub mod main_loop; pub mod process_single_media_file; diff --git a/crates/service/job/video_thumbnail_job/src/job/process_single_media_file.rs b/crates/service/job/video_thumbnail_job/src/job/process_single_media_file.rs index d57f2f62db..95424807a5 100644 --- a/crates/service/job/video_thumbnail_job/src/job/process_single_media_file.rs +++ b/crates/service/job/video_thumbnail_job/src/job/process_single_media_file.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use log::info; +use log::{error, info}; use tempdir::TempDir; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; @@ -10,6 +10,7 @@ use ffmpeg_utils::ffmpeg::ffmpeg_video_gif_preview::{ffmpeg_video_gif_preview, F use mysql_queries::queries::media_files::thumbnails::list_video_media_files_without_thumbnails_for_job::VideoMediaFileWithoutThumbnail; use mysql_queries::queries::media_files::thumbnails::update_video_media_file_with_thumbnail::update_video_media_file_with_thumbnail; +use crate::job::alert_on_error::alert_pager_and_return_err; use crate::job_dependencies::JobDependencies; /// A downloaded video file alongside its owning temp directory. @@ -32,7 +33,13 @@ pub async fn process_single_media_file( media_file.created_at, ); - let downloaded = download_video(deps, media_file).await?; + let downloaded = match download_video(deps, media_file).await { + Ok(d) => d, + Err(err) => { + error!("Failed to download video for {}: {:?}", media_file.token.as_str(), err); + return alert_pager_and_return_err(&deps.pager, err); + } + }; info!( "Downloaded video to {:?}. Generating thumbnails for {:?}.", @@ -46,53 +53,62 @@ pub async fn process_single_media_file( // Generate jpg thumbnail let jpg_path = downloaded.temp_dir.path().join("thumbnail.jpg"); - ffmpeg_video_first_frame_to_jpg_thumbnail( + if let Err(err) = ffmpeg_video_first_frame_to_jpg_thumbnail( FfmpegVideoFirstFrameToJpgThumbnailArgs { input_video_path: &downloaded.file_path, output_jpg_path: &jpg_path, }, - )?; + ) { + error!("Failed to generate JPG thumbnail for {}: {:?}", media_file.token.as_str(), err); + return alert_pager_and_return_err(&deps.pager, err.into()); + } info!("Generated JPG thumbnail for {}", media_file.token.as_str()); let jpg_object_path = format!("{video_object_path}{VIDEO_STATIC_JPG_THUMBNAIL_SUFFIX}"); - - deps - .public_bucket_client - .upload_filename(&jpg_object_path, &jpg_path) - .await?; + + if let Err(err) = deps.public_bucket_client.upload_filename(&jpg_object_path, &jpg_path).await { + error!("Failed to upload JPG thumbnail for {}: {:?}", media_file.token.as_str(), err); + return alert_pager_and_return_err(&deps.pager, err); + } info!("Uploaded JPG thumbnail to {}", jpg_object_path); // Generate gif thumbnail let gif_path = downloaded.temp_dir.path().join("thumbnail.gif"); - ffmpeg_video_gif_preview( + if let Err(err) = ffmpeg_video_gif_preview( FfmpegVideoGifPreviewArgs { input_video_path: &downloaded.file_path, output_gif_path: &gif_path, }, - )?; + ) { + error!("Failed to generate GIF preview for {}: {:?}", media_file.token.as_str(), err); + return alert_pager_and_return_err(&deps.pager, err.into()); + } info!("Generated GIF preview for {}", media_file.token.as_str()); let gif_object_path = format!("{video_object_path}{VIDEO_ANIMATED_GIF_THUMBNAIL_SUFFIX}"); - deps - .public_bucket_client - .upload_filename(&gif_object_path, &gif_path) - .await?; + if let Err(err) = deps.public_bucket_client.upload_filename(&gif_object_path, &gif_path).await { + error!("Failed to upload GIF preview for {}: {:?}", media_file.token.as_str(), err); + return alert_pager_and_return_err(&deps.pager, err); + } info!("Uploaded GIF preview to {}", gif_object_path); info!("Marking thumbnail job for {:?} done", media_file.token); // Mark the media file as having a thumbnail in the database. - update_video_media_file_with_thumbnail( + if let Err(err) = update_video_media_file_with_thumbnail( &media_file.token, CURRENT_VIDEO_THUMBNAIL_VERSION, &deps.mysql_pool, - ).await?; + ).await { + error!("Failed to update thumbnail version for {}: {:?}", media_file.token.as_str(), err); + return alert_pager_and_return_err(&deps.pager, err.into()); + } info!( "Updated thumbnail version for media file {:?} (id: {}, created at: {})",