diff --git a/src/face_processing/debug.rs b/src/face_processing/debug.rs deleted file mode 100644 index e58a718..0000000 --- a/src/face_processing/debug.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! Debug visualization functions for face processing. -//! -//! These functions create annotated images showing the processing steps, -//! useful for debugging and understanding the pipeline behavior. - -use crate::immich_api::FaceData; -use image::{DynamicImage, GenericImageView, Rgb}; -use imageproc::drawing::{draw_hollow_rect_mut, draw_line_segment_mut}; -use imageproc::rect::Rect; - -/// Draw debug visualization showing bounding box and crop region. -/// - Red rectangle: face bounding box from Immich -/// - Green rectangle: expanded crop region used for processing -/// - Red crosshair: center of the face bounding box -pub fn draw_crop_debug(img: &DynamicImage, face_data: &FaceData) -> DynamicImage { - let (img_width, img_height) = img.dimensions(); - let mut rgb_img = img.to_rgb8(); - - // Scale bounding box from metadata dimensions to actual image dimensions. - // Immich stores bounding box as pixel coordinates relative to image_width/image_height. - let scale_x = img_width as f32 / face_data.image_width as f32; - let scale_y = img_height as f32 / face_data.image_height as f32; - - let bbox_x1 = (face_data.bounding_box_x1 * scale_x) as i32; - let bbox_y1 = (face_data.bounding_box_y1 * scale_y) as i32; - let bbox_x2 = (face_data.bounding_box_x2 * scale_x) as i32; - let bbox_y2 = (face_data.bounding_box_y2 * scale_y) as i32; - - let face_width = (bbox_x2 - bbox_x1) as u32; - let face_height = (bbox_y2 - bbox_y1) as u32; - - // Calculate crop region (same logic as crop_face_with_intermediate) - let face_size = face_width.max(face_height); - let padding = face_size / 2; - let crop_size = face_size + padding * 2; - - let center_x = (bbox_x1 + bbox_x2) / 2; - let center_y = (bbox_y1 + bbox_y2) / 2; - - let crop_x1 = (center_x - crop_size as i32 / 2).max(0) as u32; - let crop_y1 = (center_y - crop_size as i32 / 2).max(0) as u32; - let crop_x1 = crop_x1.min(img_width.saturating_sub(crop_size)); - let crop_y1 = crop_y1.min(img_height.saturating_sub(crop_size)); - let actual_crop_size = crop_size.min(img_width - crop_x1).min(img_height - crop_y1); - - // Colors - let red = Rgb([255u8, 0, 0]); - let green = Rgb([0u8, 255, 0]); - - // Draw face bounding box (red) - draw multiple times for thickness - for offset in 0..3i32 { - let rect = Rect::at(bbox_x1 - offset, bbox_y1 - offset) - .of_size(face_width + offset as u32 * 2, face_height + offset as u32 * 2); - draw_hollow_rect_mut(&mut rgb_img, rect, red); - } - - // Draw crop region (green) - draw multiple times for thickness - for offset in 0..3i32 { - let rect = Rect::at(crop_x1 as i32 - offset, crop_y1 as i32 - offset) - .of_size( - actual_crop_size + offset as u32 * 2, - actual_crop_size + offset as u32 * 2, - ); - draw_hollow_rect_mut(&mut rgb_img, rect, green); - } - - // Draw crosshair at face center - let cross_size = 20i32; - draw_line_segment_mut( - &mut rgb_img, - ((center_x - cross_size) as f32, center_y as f32), - ((center_x + cross_size) as f32, center_y as f32), - red, - ); - draw_line_segment_mut( - &mut rgb_img, - (center_x as f32, (center_y - cross_size) as f32), - (center_x as f32, (center_y + cross_size) as f32), - red, - ); - - DynamicImage::ImageRgb8(rgb_img) -} - -// Future debug visualization functions: -// - draw_landmarks_debug: Face with 68-point landmarks drawn -// - draw_alignment_debug: Before/after alignment visualization diff --git a/src/face_processing/mod.rs b/src/face_processing/mod.rs deleted file mode 100644 index 3d93897..0000000 --- a/src/face_processing/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! Image processing module. -//! -//! This module handles face detection, landmark detection, -//! alignment, and image transformation. - -mod crop; -pub mod debug; -mod orientation; -pub mod types; - -pub use crop::*; -pub use orientation::load_image_with_orientation; -pub use types::*; - -// TODO: Implement these modules -// mod landmarks; -// mod alignment; -// mod filters; diff --git a/src/lib.rs b/src/lib.rs index 7f517f0..4ef4ae2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,7 +7,6 @@ pub mod config; pub mod error; -pub mod face_processing; pub mod immich_api; pub mod job; pub mod models; diff --git a/src/models/dlib_landmarks.rs b/src/models/dlib_landmarks.rs index 305840e..dbabca6 100644 --- a/src/models/dlib_landmarks.rs +++ b/src/models/dlib_landmarks.rs @@ -4,7 +4,7 @@ //! in a thread-safe singleton to avoid reloading the model for every image. use crate::error::{Error, Result}; -use crate::face_processing::types::{Landmarks, Point}; +use crate::pipeline::{Landmarks, Point}; use dlib_face_recognition::{ FaceDetector, FaceDetectorTrait, ImageMatrix, LandmarkPredictor, LandmarkPredictorTrait, Rectangle, diff --git a/src/models/dmhead.rs b/src/models/dmhead.rs index b87bf94..6f732f4 100644 --- a/src/models/dmhead.rs +++ b/src/models/dmhead.rs @@ -8,7 +8,7 @@ //! Output: [yaw, pitch, roll] in degrees use crate::error::{Error, Result}; -use crate::face_processing::types::HeadPose; +use crate::pipeline::HeadPose; use image::DynamicImage; use ndarray::Array4; use ort::session::{builder::GraphOptimizationLevel, Session}; diff --git a/src/face_processing/crop.rs b/src/pipeline/crop_utils.rs similarity index 98% rename from src/face_processing/crop.rs rename to src/pipeline/crop_utils.rs index 47c823f..03f27d3 100644 --- a/src/face_processing/crop.rs +++ b/src/pipeline/crop_utils.rs @@ -3,8 +3,8 @@ //! Extracts and resizes face regions from images using bounding box data. use crate::error::{Error, Result}; -use crate::face_processing::types::BoundingBox; use crate::immich_api::FaceData; +use crate::pipeline::BoundingBox; use image::imageops::FilterType; use image::{DynamicImage, GenericImageView}; diff --git a/src/pipeline/mod.rs b/src/pipeline/mod.rs index 141aa40..753354e 100644 --- a/src/pipeline/mod.rs +++ b/src/pipeline/mod.rs @@ -13,10 +13,16 @@ //! let result = pipeline.execute(ctx, &config, &cancel_token, &skip_stats).await; //! ``` +mod crop_utils; +mod orientation; mod traits; +mod types; pub mod steps; +pub use crop_utils::{crop_face_with_intermediate, CropResult}; +pub use orientation::load_image_with_orientation; pub use traits::*; +pub use types::*; use crate::config::Config; use crate::web::AtomicSkipStats; @@ -87,14 +93,18 @@ impl Pipeline { /// Create the default processing pipeline with standard steps. /// /// Pipeline order: - /// 1. FaceResolutionStep - Validate face size from Immich metadata - /// 2. DecodeImageStep - Load and orient the image - /// 3. BrightnessStep - Filter by luminance - /// 4. CropFaceStep - Extract face region with padding - /// 5. HeadPoseStep - Filter non-frontal faces (DMHead) - /// 6. LandmarksStep - Detect 68 facial landmarks (dlib) - /// 7. AlignmentStep - Align face based on eye positions - /// 8. ResizeStep - Final resize to output size + /// 1. FaceResolutionStep - Validate face size from Immich metadata (Validator) + /// 2. DecodeImageStep - Load and orient the image (Transform) + /// 3. BrightnessStep - Filter by luminance (Validator) + /// 4. CropFaceStep - Extract face region with padding (Transform) + /// 5. HeadPoseStep - Filter non-frontal faces (Validator) + /// 6. LandmarksStep - Detect 68 facial landmarks (Detector) + /// 7. EyeFilterStep - Filter closed eyes by EAR (Validator) + /// 8. AlignmentStep - Align face based on eye positions (Transform) + /// 9. ResizeStep - Final resize to output size (Transform) + /// + /// Debug visualizations are only generated for validator steps: + /// BrightnessStep, HeadPoseStep, EyeFilterStep pub fn with_default_steps() -> Self { use steps::*; @@ -105,6 +115,7 @@ impl Pipeline { pipeline.add_step(Box::new(CropFaceStep)); pipeline.add_step(Box::new(HeadPoseStep)); pipeline.add_step(Box::new(LandmarksStep)); + pipeline.add_step(Box::new(EyeFilterStep)); pipeline.add_step(Box::new(AlignmentStep)); pipeline.add_step(Box::new(ResizeStep)); pipeline @@ -149,7 +160,7 @@ impl Pipeline { // Generate debug visualization if enabled (step passed) if config.processing.output.keep_intermediates { - if let Some(debug_img) = step.debug_visualize(&ctx) { + if let Some(debug_img) = step.debug_visualize(&ctx, config) { ctx.add_debug_image(step.id(), debug_img, true); } } @@ -160,7 +171,7 @@ impl Pipeline { // Generate debug visualization for the failing step if enabled if config.processing.output.keep_intermediates { - if let Some(debug_img) = step.debug_visualize(&ctx) { + if let Some(debug_img) = step.debug_visualize(&ctx, config) { ctx.add_debug_image(step.id(), debug_img, false); } } @@ -285,6 +296,7 @@ mod tests { assert!(ids.contains(&"crop")); assert!(ids.contains(&"head_pose")); assert!(ids.contains(&"landmarks")); + assert!(ids.contains(&"eye_filter")); assert!(ids.contains(&"alignment")); assert!(ids.contains(&"resize")); } diff --git a/src/face_processing/orientation.rs b/src/pipeline/orientation.rs similarity index 100% rename from src/face_processing/orientation.rs rename to src/pipeline/orientation.rs diff --git a/src/pipeline/steps/alignment.rs b/src/pipeline/steps/alignment.rs index 67de80b..845cb00 100644 --- a/src/pipeline/steps/alignment.rs +++ b/src/pipeline/steps/alignment.rs @@ -4,10 +4,9 @@ //! across all images in the timelapse. use crate::config::Config; -use crate::face_processing::types::Point; -use crate::pipeline::{PipelineContext, ProcessingStep, StepOutcome}; +use crate::pipeline::{Point, PipelineContext, ProcessingStep, StepOutcome}; use async_trait::async_trait; -use image::{DynamicImage, GenericImageView, Rgb, RgbImage}; +use image::{DynamicImage, GenericImageView, Rgb}; use imageproc::geometric_transformations::{rotate_about_center, Interpolation}; /// Aligns faces based on eye positions. @@ -36,7 +35,7 @@ impl ProcessingStep for AlignmentStep { } // Get landmarks from previous step - let landmarks: crate::face_processing::types::Landmarks = match ctx + let landmarks: crate::pipeline::Landmarks = match ctx .get_computed("landmarks") .and_then(|v| v.as_landmarks()) { @@ -156,72 +155,13 @@ impl ProcessingStep for AlignmentStep { StepOutcome::Continue(ctx) } - - fn debug_visualize(&self, ctx: &PipelineContext) -> Option { - // Get landmarks for visualization - let landmarks: &crate::face_processing::types::Landmarks = - ctx.get_computed("landmarks").and_then(|v| v.as_landmarks())?; - let image = ctx.image.as_ref()?; - - let mut debug_img = image.to_rgb8(); - let (width, height) = (debug_img.width(), debug_img.height()); - - // Draw target eye positions - let output_size = width; // Assuming square output - let eye_y = (output_size as f32 * 0.35) as u32; // Default eye_y_position - - // Draw horizontal line at target eye Y position - for x in 0..width { - if eye_y < height { - debug_img.put_pixel(x, eye_y, Rgb([0, 255, 0])); - } - } - - // Draw vertical lines at target eye X positions (assuming 0.3 inter_eye_distance) - let inter_eye = (output_size as f32 * 0.3) as u32; - let left_x = (width - inter_eye) / 2; - let right_x = left_x + inter_eye; - - for y in 0..height { - if left_x < width { - debug_img.put_pixel(left_x, y, Rgb([0, 255, 0])); - } - if right_x < width { - debug_img.put_pixel(right_x, y, Rgb([0, 255, 0])); - } - } - - // Draw actual eye positions - let left_eye = landmarks.left_eye_center(); - let right_eye = landmarks.right_eye_center(); - - draw_marker(&mut debug_img, left_eye.x as u32, left_eye.y as u32, Rgb([255, 0, 0])); - draw_marker(&mut debug_img, right_eye.x as u32, right_eye.y as u32, Rgb([255, 0, 0])); - - Some(DynamicImage::ImageRgb8(debug_img)) - } -} - -/// Draw a marker (small filled square) at the given position. -fn draw_marker(img: &mut RgbImage, x: u32, y: u32, color: Rgb) { - let (width, height) = (img.width(), img.height()); - let size: i32 = 3; - - for dy in 0..=size * 2 { - for dx in 0..=size * 2 { - let px = (x as i32 + dx - size) as u32; - let py = (y as i32 + dy - size) as u32; - if px < width && py < height { - img.put_pixel(px, py, color); - } - } - } } #[cfg(test)] mod tests { use super::*; use crate::immich_api::FaceData; + use image::RgbImage; fn make_test_ctx() -> PipelineContext { let face_data = FaceData { diff --git a/src/pipeline/steps/brightness.rs b/src/pipeline/steps/brightness.rs index fc3b972..b97fe30 100644 --- a/src/pipeline/steps/brightness.rs +++ b/src/pipeline/steps/brightness.rs @@ -95,7 +95,7 @@ impl ProcessingStep for BrightnessStep { StepOutcome::Continue(ctx) } - fn debug_visualize(&self, ctx: &PipelineContext) -> Option { + fn debug_visualize(&self, ctx: &PipelineContext, _config: &Config) -> Option { // Get brightness from computed values let brightness = ctx .get_computed("brightness") diff --git a/src/pipeline/steps/crop.rs b/src/pipeline/steps/crop.rs index b62ebe1..ffe00fd 100644 --- a/src/pipeline/steps/crop.rs +++ b/src/pipeline/steps/crop.rs @@ -3,11 +3,9 @@ //! Crops the face region from the full image using the bounding box data. use crate::config::Config; -use crate::face_processing::crop_face_with_intermediate; -use crate::face_processing::debug::draw_crop_debug; +use crate::pipeline::crop_face_with_intermediate; use crate::pipeline::{ComputedValue, PipelineContext, ProcessingStep, StepOutcome}; use async_trait::async_trait; -use image::DynamicImage; /// Crops the face region from the full image. /// @@ -49,30 +47,6 @@ impl ProcessingStep for CropFaceStep { } } - fn debug_visualize(&self, ctx: &PipelineContext) -> Option { - // Draw the crop region on the original image - // Note: This requires access to the original image before cropping, - // which we don't have here. For now, we'll create the debug image - // during execution if needed. This is a limitation of the current - // design that could be addressed by storing the original image. - - // For now, return None and handle debug visualization in the pipeline - // execution or via a separate mechanism - ctx.raw_bytes.as_ref()?; - - // If we had the original image, we could do: - // Some(draw_crop_debug(&original, &ctx.face_data)) - None - } -} - -/// Generates a debug visualization of the crop region. -/// -/// This can be called separately before the crop step to visualize -/// what will be cropped. -#[allow(dead_code)] -pub fn generate_crop_debug(image: &DynamicImage, ctx: &PipelineContext) -> DynamicImage { - draw_crop_debug(image, &ctx.face_data) } #[cfg(test)] diff --git a/src/pipeline/steps/decode.rs b/src/pipeline/steps/decode.rs index 6c73f5c..cffd852 100644 --- a/src/pipeline/steps/decode.rs +++ b/src/pipeline/steps/decode.rs @@ -3,7 +3,7 @@ //! Decodes raw image bytes and applies EXIF orientation correction. use crate::config::Config; -use crate::face_processing::load_image_with_orientation; +use crate::pipeline::load_image_with_orientation; use crate::pipeline::{PipelineContext, ProcessingStep, StepOutcome}; use async_trait::async_trait; diff --git a/src/pipeline/steps/eye_filter.rs b/src/pipeline/steps/eye_filter.rs new file mode 100644 index 0000000..d9508c9 --- /dev/null +++ b/src/pipeline/steps/eye_filter.rs @@ -0,0 +1,296 @@ +//! Eye aspect ratio (EAR) filter step. +//! +//! Filters images based on eye openness using the Eye Aspect Ratio computed +//! from facial landmarks. + +use crate::config::Config; +use crate::pipeline::{Landmarks, PipelineContext, ProcessingStep, StepOutcome}; +use async_trait::async_trait; +use image::{DynamicImage, Rgb, RgbImage}; + +/// Filters images where eyes appear closed based on Eye Aspect Ratio. +/// +/// This validator step reads the EAR value computed by LandmarksStep +/// and skips images where the average EAR is below the configured threshold. +/// +/// Must run after LandmarksStep. +pub struct EyeFilterStep; + +#[async_trait] +impl ProcessingStep for EyeFilterStep { + fn id(&self) -> &'static str { + "eye_filter" + } + + fn name(&self) -> &'static str { + "Eye Filter" + } + + async fn execute(&self, ctx: PipelineContext, config: &Config) -> StepOutcome { + // Skip if eye filtering is disabled + if !config.processing.eye_filter.enabled { + return StepOutcome::Continue(ctx); + } + + // Get EAR from computed values (set by LandmarksStep) + let avg_ear = match ctx.get_computed("ear").and_then(|v| v.as_float()) { + Some(ear) => ear, + None => { + // No EAR available - landmarks step must have been skipped + tracing::warn!("EAR not available for eye filter, skipping check"); + return StepOutcome::Continue(ctx); + } + }; + + let min_ear = config.processing.eye_filter.min_ear; + + if avg_ear < min_ear { + return StepOutcome::Skip { + ctx, + reason: "eyes_closed".to_string(), + detail: Some(format!("EAR {:.3} below threshold {:.3}", avg_ear, min_ear)), + }; + } + + tracing::trace!("Eye filter passed: EAR {:.3} >= {:.3}", avg_ear, min_ear); + + StepOutcome::Continue(ctx) + } + + fn debug_visualize(&self, ctx: &PipelineContext, _config: &Config) -> Option { + // Get landmarks for eye visualization + let landmarks: &Landmarks = ctx + .get_computed("landmarks") + .and_then(|v| v.as_landmarks())?; + + // Get EAR values + let ear = landmarks.eye_aspect_ratio(); + let avg_ear = (ear.left + ear.right) / 2.0; + + // Get the current image to draw on + let image = ctx.image.as_ref()?; + let rgb = image.to_rgb8(); + let (width, height) = (rgb.width(), rgb.height()); + + // Create a copy for visualization + let mut debug_img = rgb.clone(); + + // Draw eye landmarks (points 36-47) + let points = landmarks.points(); + + // Left eye (points 36-41) - yellow or red depending on EAR + let left_color = if ear.left >= 0.2 { + Rgb([0, 255, 0]) // Green - open + } else { + Rgb([255, 0, 0]) // Red - closed + }; + + for i in 36..42 { + let point = &points[i]; + draw_cross(&mut debug_img, point.x as u32, point.y as u32, left_color); + } + + // Right eye (points 42-47) + let right_color = if ear.right >= 0.2 { + Rgb([0, 255, 0]) // Green - open + } else { + Rgb([255, 0, 0]) // Red - closed + }; + + for i in 42..48 { + let point = &points[i]; + draw_cross(&mut debug_img, point.x as u32, point.y as u32, right_color); + } + + // Draw eye centers + let left_eye = landmarks.left_eye_center(); + let right_eye = landmarks.right_eye_center(); + draw_marker(&mut debug_img, left_eye.x as u32, left_eye.y as u32, Rgb([0, 255, 255])); + draw_marker(&mut debug_img, right_eye.x as u32, right_eye.y as u32, Rgb([0, 255, 255])); + + // Draw info bar at bottom + let bar_height = 20u32; + let bar_y = height.saturating_sub(bar_height); + + // Draw background + for y in bar_y..height { + for x in 0..width { + debug_img.put_pixel(x, y, Rgb([0, 0, 0])); + } + } + + // Draw EAR values + let text = format!( + "L:{:.2} R:{:.2} Avg:{:.2}", + ear.left, ear.right, avg_ear + ); + draw_simple_text(&mut debug_img, 5, bar_y + 6, &text, Rgb([255, 255, 255])); + + Some(DynamicImage::ImageRgb8(debug_img)) + } +} + +/// Draw a small cross at the given position. +fn draw_cross(img: &mut RgbImage, x: u32, y: u32, color: Rgb) { + let (width, height) = (img.width(), img.height()); + let size: i32 = 2; + + // Check base coordinates are in bounds + if x >= width || y >= height { + return; + } + + for dx in 0..=size * 2 { + let px = (x as i32 + dx - size) as u32; + if px < width && y < height { + img.put_pixel(px, y, color); + } + } + for dy in 0..=size * 2 { + let py = (y as i32 + dy - size) as u32; + if x < width && py < height { + img.put_pixel(x, py, color); + } + } +} + +/// Draw a marker (small filled square) at the given position. +fn draw_marker(img: &mut RgbImage, x: u32, y: u32, color: Rgb) { + let (width, height) = (img.width(), img.height()); + let size: i32 = 3; + + for dy in 0..=size * 2 { + for dx in 0..=size * 2 { + let px = (x as i32 + dx - size) as u32; + let py = (y as i32 + dy - size) as u32; + if px < width && py < height { + img.put_pixel(px, py, color); + } + } + } +} + +/// Draw simple text using a basic 5x7 pixel font. +fn draw_simple_text(img: &mut RgbImage, x: u32, y: u32, text: &str, color: Rgb) { + let (width, height) = (img.width(), img.height()); + let mut cursor_x = x; + + for ch in text.chars() { + let pattern = get_char_pattern(ch); + for (row_idx, row) in pattern.iter().enumerate() { + for col in 0..5 { + if (row >> (4 - col)) & 1 == 1 { + let px = cursor_x + col; + let py = y + row_idx as u32; + if px < width && py < height { + img.put_pixel(px, py, color); + } + } + } + } + cursor_x += 6; // 5 pixels wide + 1 pixel spacing + } +} + +/// Get a 5x7 pixel pattern for a character. +fn get_char_pattern(ch: char) -> [u8; 7] { + match ch { + '0' => [0b01110, 0b10001, 0b10011, 0b10101, 0b11001, 0b10001, 0b01110], + '1' => [0b00100, 0b01100, 0b00100, 0b00100, 0b00100, 0b00100, 0b01110], + '2' => [0b01110, 0b10001, 0b00001, 0b00110, 0b01000, 0b10000, 0b11111], + '3' => [0b01110, 0b10001, 0b00001, 0b00110, 0b00001, 0b10001, 0b01110], + '4' => [0b00010, 0b00110, 0b01010, 0b10010, 0b11111, 0b00010, 0b00010], + '5' => [0b11111, 0b10000, 0b11110, 0b00001, 0b00001, 0b10001, 0b01110], + '6' => [0b00110, 0b01000, 0b10000, 0b11110, 0b10001, 0b10001, 0b01110], + '7' => [0b11111, 0b00001, 0b00010, 0b00100, 0b01000, 0b01000, 0b01000], + '8' => [0b01110, 0b10001, 0b10001, 0b01110, 0b10001, 0b10001, 0b01110], + '9' => [0b01110, 0b10001, 0b10001, 0b01111, 0b00001, 0b00010, 0b01100], + 'L' => [0b10000, 0b10000, 0b10000, 0b10000, 0b10000, 0b10000, 0b11111], + 'R' => [0b11110, 0b10001, 0b10001, 0b11110, 0b10100, 0b10010, 0b10001], + 'A' => [0b01110, 0b10001, 0b10001, 0b11111, 0b10001, 0b10001, 0b10001], + 'v' => [0b00000, 0b00000, 0b10001, 0b10001, 0b10001, 0b01010, 0b00100], + 'g' => [0b00000, 0b00000, 0b01111, 0b10001, 0b01111, 0b00001, 0b01110], + ':' => [0b00000, 0b00100, 0b00000, 0b00000, 0b00100, 0b00000, 0b00000], + '.' => [0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00100], + ' ' => [0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00000], + _ => [0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00000], + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::immich_api::FaceData; + use crate::pipeline::ComputedValue; + + fn make_test_ctx() -> PipelineContext { + let face_data = FaceData { + bounding_box_x1: 0.0, + bounding_box_y1: 0.0, + bounding_box_x2: 100.0, + bounding_box_y2: 100.0, + image_width: 100, + image_height: 100, + }; + PipelineContext::new("test".to_string(), "2024-01-01".to_string(), face_data) + } + + #[tokio::test] + async fn test_disabled_skips_check() { + let step = EyeFilterStep; + let mut ctx = make_test_ctx(); + ctx.set_computed("ear", ComputedValue::Float(0.1)); // Below threshold + let mut config = Config::default(); + config.processing.eye_filter.enabled = false; + + match step.execute(ctx, &config).await { + StepOutcome::Continue(_) => {} // Expected + other => panic!("Expected Continue when disabled, got {:?}", other), + } + } + + #[tokio::test] + async fn test_below_threshold_skips() { + let step = EyeFilterStep; + let mut ctx = make_test_ctx(); + ctx.set_computed("ear", ComputedValue::Float(0.1)); // Below default 0.2 threshold + let mut config = Config::default(); + config.processing.eye_filter.enabled = true; + config.processing.eye_filter.min_ear = 0.2; + + match step.execute(ctx, &config).await { + StepOutcome::Skip { reason, .. } => { + assert_eq!(reason, "eyes_closed"); + } + other => panic!("Expected Skip, got {:?}", other), + } + } + + #[tokio::test] + async fn test_above_threshold_continues() { + let step = EyeFilterStep; + let mut ctx = make_test_ctx(); + ctx.set_computed("ear", ComputedValue::Float(0.3)); // Above threshold + let mut config = Config::default(); + config.processing.eye_filter.enabled = true; + config.processing.eye_filter.min_ear = 0.2; + + match step.execute(ctx, &config).await { + StepOutcome::Continue(_) => {} // Expected + other => panic!("Expected Continue, got {:?}", other), + } + } + + #[tokio::test] + async fn test_no_ear_continues() { + let step = EyeFilterStep; + let ctx = make_test_ctx(); // No EAR set + let mut config = Config::default(); + config.processing.eye_filter.enabled = true; + + match step.execute(ctx, &config).await { + StepOutcome::Continue(_) => {} // Expected - gracefully handles missing EAR + other => panic!("Expected Continue when no EAR, got {:?}", other), + } + } +} diff --git a/src/pipeline/steps/head_pose.rs b/src/pipeline/steps/head_pose.rs index a2d38c6..6ff3509 100644 --- a/src/pipeline/steps/head_pose.rs +++ b/src/pipeline/steps/head_pose.rs @@ -136,7 +136,7 @@ impl ProcessingStep for HeadPoseStep { StepOutcome::Continue(ctx) } - fn debug_visualize(&self, ctx: &PipelineContext) -> Option { + fn debug_visualize(&self, ctx: &PipelineContext, _config: &Config) -> Option { // Get head pose from computed values let pose = ctx .get_computed("head_pose") diff --git a/src/pipeline/steps/landmarks.rs b/src/pipeline/steps/landmarks.rs index ab1dffd..6d182b0 100644 --- a/src/pipeline/steps/landmarks.rs +++ b/src/pipeline/steps/landmarks.rs @@ -3,20 +3,19 @@ //! Uses dlib to detect 68 facial landmarks for alignment and eye filtering. use crate::config::Config; -use crate::face_processing::types::Landmarks; use crate::models::DlibLandmarks; -use crate::pipeline::{ComputedValue, PipelineContext, ProcessingStep, StepOutcome}; +use crate::pipeline::{ComputedValue, Landmarks, PipelineContext, ProcessingStep, StepOutcome}; use async_trait::async_trait; -use image::{DynamicImage, Rgb, RgbImage}; use tokio::task; -/// Detects facial landmarks and optionally filters based on eye aspect ratio. +/// Detects facial landmarks using dlib. /// /// This step: /// 1. Uses dlib to detect faces and 68 landmarks /// 2. Stores Landmarks in ctx.computed["landmarks"] /// 3. Computes EAR and stores in ctx.computed["ear"] -/// 4. Optionally skips if EAR is below threshold (eyes closed) +/// +/// Eye filtering (skipping closed eyes) is handled by EyeFilterStep. pub struct LandmarksStep; #[async_trait] @@ -97,18 +96,6 @@ impl ProcessingStep for LandmarksStep { // Store landmarks ctx.set_computed("landmarks", ComputedValue::Landmarks(Box::new(landmarks))); - // Check eye filter if enabled - if config.processing.eye_filter.enabled { - let min_ear = config.processing.eye_filter.min_ear; - if avg_ear < min_ear { - return StepOutcome::Skip { - ctx, - reason: "eyes_closed".to_string(), - detail: Some(format!("EAR {:.3} below threshold {:.3}", avg_ear, min_ear)), - }; - } - } - tracing::trace!( "Landmarks detected: EAR left={:.3}, right={:.3}, avg={:.3}", ear.left, @@ -118,87 +105,13 @@ impl ProcessingStep for LandmarksStep { StepOutcome::Continue(ctx) } - - fn debug_visualize(&self, ctx: &PipelineContext) -> Option { - // Get landmarks from computed values - let landmarks: &Landmarks = ctx - .get_computed("landmarks") - .and_then(|v| v.as_landmarks())?; - - // Get the current image to draw on - let image = ctx.image.as_ref()?; - let mut debug_img = image.to_rgb8(); - - // Draw all 68 landmark points - let points = landmarks.points(); - for (i, point) in points.iter().enumerate() { - let x = point.x as u32; - let y = point.y as u32; - - // Color-code different facial regions - let color = match i { - 0..=16 => Rgb([255, 0, 0]), // Jaw (red) - 17..=21 => Rgb([0, 255, 0]), // Left eyebrow (green) - 22..=26 => Rgb([0, 255, 0]), // Right eyebrow (green) - 27..=35 => Rgb([0, 0, 255]), // Nose (blue) - 36..=41 => Rgb([255, 255, 0]), // Left eye (yellow) - 42..=47 => Rgb([255, 255, 0]), // Right eye (yellow) - 48..=67 => Rgb([255, 0, 255]), // Mouth (magenta) - _ => Rgb([255, 255, 255]), // Other (white) - }; - - // Draw a small cross at each point - draw_cross(&mut debug_img, x, y, color); - } - - // Draw eye centers - let left_eye = landmarks.left_eye_center(); - let right_eye = landmarks.right_eye_center(); - draw_cross( - &mut debug_img, - left_eye.x as u32, - left_eye.y as u32, - Rgb([0, 255, 255]), - ); - draw_cross( - &mut debug_img, - right_eye.x as u32, - right_eye.y as u32, - Rgb([0, 255, 255]), - ); - - Some(DynamicImage::ImageRgb8(debug_img)) - } -} - -/// Draw a small cross at the given position. -fn draw_cross(img: &mut RgbImage, x: u32, y: u32, color: Rgb) { - let (width, height) = (img.width(), img.height()); - let size: i32 = 2; - - // Check base coordinates are in bounds - if x >= width || y >= height { - return; - } - - for dx in 0..=size * 2 { - let px = (x as i32 + dx - size) as u32; - if px < width && y < height { - img.put_pixel(px, y, color); - } - } - for dy in 0..=size * 2 { - let py = (y as i32 + dy - size) as u32; - if x < width && py < height { - img.put_pixel(x, py, color); - } - } } #[cfg(test)] mod tests { use super::*; use crate::immich_api::FaceData; + use image::{DynamicImage, RgbImage}; fn make_test_ctx() -> PipelineContext { let face_data = FaceData { diff --git a/src/pipeline/steps/mod.rs b/src/pipeline/steps/mod.rs index 4cb381d..706c3df 100644 --- a/src/pipeline/steps/mod.rs +++ b/src/pipeline/steps/mod.rs @@ -7,6 +7,7 @@ mod alignment; mod brightness; mod crop; mod decode; +mod eye_filter; mod face_resolution; mod head_pose; mod landmarks; @@ -16,6 +17,7 @@ pub use alignment::AlignmentStep; pub use brightness::BrightnessStep; pub use crop::CropFaceStep; pub use decode::DecodeImageStep; +pub use eye_filter::EyeFilterStep; pub use face_resolution::FaceResolutionStep; pub use head_pose::HeadPoseStep; pub use landmarks::LandmarksStep; diff --git a/src/pipeline/traits.rs b/src/pipeline/traits.rs index 95131df..aabfa50 100644 --- a/src/pipeline/traits.rs +++ b/src/pipeline/traits.rs @@ -4,7 +4,7 @@ //! extensible image processing pipelines. use crate::config::Config; -use crate::face_processing::types::{BoundingBox, HeadPose, Landmarks}; +use crate::pipeline::types::{BoundingBox, HeadPose, Landmarks}; use crate::immich_api::FaceData; use async_trait::async_trait; use bytes::Bytes; @@ -243,7 +243,9 @@ pub trait ProcessingStep: Send + Sync { /// /// Called after `execute()` if debug mode is enabled and the step /// returned `Continue`. The returned image is saved to the debug folder. - fn debug_visualize(&self, _ctx: &PipelineContext) -> Option { + /// + /// Steps should return `None` if they are disabled (check config.processing.X.enabled). + fn debug_visualize(&self, _ctx: &PipelineContext, _config: &Config) -> Option { None } } diff --git a/src/face_processing/types.rs b/src/pipeline/types.rs similarity index 100% rename from src/face_processing/types.rs rename to src/pipeline/types.rs diff --git a/src/web/handlers/config.rs b/src/web/handlers/config.rs index 44b3dfe..d2cbe79 100644 --- a/src/web/handlers/config.rs +++ b/src/web/handlers/config.rs @@ -1,8 +1,8 @@ //! Configuration endpoints. use crate::config::{ - AlignmentConfig, BrightnessConfig, FaceResolutionConfig, OutputConfig, ProcessingConfig, - VideoConfig, + AlignmentConfig, BrightnessConfig, EyeFilterConfig, FaceResolutionConfig, HeadPoseConfig, + OutputConfig, ProcessingConfig, VideoConfig, }; use crate::web::state::AppState; use axum::{extract::State, http::StatusCode, response::Json}; @@ -40,6 +40,8 @@ pub struct ProcessingConfigUpdate { pub max_workers: Option, pub face_resolution: Option, pub brightness: Option, + pub head_pose: Option, + pub eye_filter: Option, pub output: Option, pub alignment: Option, } @@ -117,6 +119,38 @@ fn validate_processing_config(proc: &ProcessingConfigUpdate) -> Result<(), Valid } } + if let Some(ref hp) = proc.head_pose { + if hp.enabled { + if hp.max_yaw < 0.0 || hp.max_yaw > 90.0 { + return Err(ValidationError::new( + "processing.head_pose.max_yaw", + format!("must be between 0 and 90, got {}", hp.max_yaw), + )); + } + if hp.max_pitch < 0.0 || hp.max_pitch > 90.0 { + return Err(ValidationError::new( + "processing.head_pose.max_pitch", + format!("must be between 0 and 90, got {}", hp.max_pitch), + )); + } + if hp.max_roll < 0.0 || hp.max_roll > 90.0 { + return Err(ValidationError::new( + "processing.head_pose.max_roll", + format!("must be between 0 and 90, got {}", hp.max_roll), + )); + } + } + } + + if let Some(ref ef) = proc.eye_filter { + if ef.enabled && !(0.0..=0.5).contains(&ef.min_ear) { + return Err(ValidationError::new( + "processing.eye_filter.min_ear", + format!("must be between 0.0 and 0.5, got {}", ef.min_ear), + )); + } + } + if let Some(ref out) = proc.output { if out.size < 64 { return Err(ValidationError::new( @@ -211,6 +245,12 @@ pub async fn update_config( if let Some(v) = proc.brightness { config.processing.brightness = v; } + if let Some(v) = proc.head_pose { + config.processing.head_pose = v; + } + if let Some(v) = proc.eye_filter { + config.processing.eye_filter = v; + } if let Some(v) = proc.output { config.processing.output = v; }