danvics/immich-automated-selfie-timelapse
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Removed the face processing module, moved it to the pipeline module

Browse source
This commit is contained in:
Arnaud_Cayrol 2026-02-03 22:55:59 +01:00
parent 0114426172
commit 1767096d00
19 changed files with 382 additions and 309 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
src/face_processing/debug.rs
View file

@ -1,87 +0,0 @@
//! Debug visualization functions for face processing.
//!
//! These functions create annotated images showing the processing steps,
//! useful for debugging and understanding the pipeline behavior.
use crate::immich_api::FaceData;
use image::{DynamicImage, GenericImageView, Rgb};
use imageproc::drawing::{draw_hollow_rect_mut, draw_line_segment_mut};
use imageproc::rect::Rect;
/// Draw debug visualization showing bounding box and crop region.
/// - Red rectangle: face bounding box from Immich
/// - Green rectangle: expanded crop region used for processing
/// - Red crosshair: center of the face bounding box
pub fn draw_crop_debug(img: &DynamicImage, face_data: &FaceData) -> DynamicImage {
let (img_width, img_height) = img.dimensions();
let mut rgb_img = img.to_rgb8();
// Scale bounding box from metadata dimensions to actual image dimensions.
// Immich stores bounding box as pixel coordinates relative to image_width/image_height.
let scale_x = img_width as f32 / face_data.image_width as f32;
let scale_y = img_height as f32 / face_data.image_height as f32;
let bbox_x1 = (face_data.bounding_box_x1 * scale_x) as i32;
let bbox_y1 = (face_data.bounding_box_y1 * scale_y) as i32;
let bbox_x2 = (face_data.bounding_box_x2 * scale_x) as i32;
let bbox_y2 = (face_data.bounding_box_y2 * scale_y) as i32;
let face_width = (bbox_x2 - bbox_x1) as u32;
let face_height = (bbox_y2 - bbox_y1) as u32;
// Calculate crop region (same logic as crop_face_with_intermediate)
let face_size = face_width.max(face_height);
let padding = face_size / 2;
let crop_size = face_size + padding * 2;
let center_x = (bbox_x1 + bbox_x2) / 2;
let center_y = (bbox_y1 + bbox_y2) / 2;
let crop_x1 = (center_x - crop_size as i32 / 2).max(0) as u32;
let crop_y1 = (center_y - crop_size as i32 / 2).max(0) as u32;
let crop_x1 = crop_x1.min(img_width.saturating_sub(crop_size));
let crop_y1 = crop_y1.min(img_height.saturating_sub(crop_size));
let actual_crop_size = crop_size.min(img_width - crop_x1).min(img_height - crop_y1);
// Colors
let red = Rgb([255u8, 0, 0]);
let green = Rgb([0u8, 255, 0]);
// Draw face bounding box (red) - draw multiple times for thickness
for offset in 0..3i32 {
let rect = Rect::at(bbox_x1 - offset, bbox_y1 - offset)
.of_size(face_width + offset as u32 * 2, face_height + offset as u32 * 2);
draw_hollow_rect_mut(&mut rgb_img, rect, red);
}
// Draw crop region (green) - draw multiple times for thickness
for offset in 0..3i32 {
let rect = Rect::at(crop_x1 as i32 - offset, crop_y1 as i32 - offset)
.of_size(
actual_crop_size + offset as u32 * 2,
actual_crop_size + offset as u32 * 2,
);
draw_hollow_rect_mut(&mut rgb_img, rect, green);
}
// Draw crosshair at face center
let cross_size = 20i32;
draw_line_segment_mut(
&mut rgb_img,
((center_x - cross_size) as f32, center_y as f32),
((center_x + cross_size) as f32, center_y as f32),
red,
);
draw_line_segment_mut(
&mut rgb_img,
(center_x as f32, (center_y - cross_size) as f32),
(center_x as f32, (center_y + cross_size) as f32),
red,
);
DynamicImage::ImageRgb8(rgb_img)
}
// Future debug visualization functions:
// - draw_landmarks_debug: Face with 68-point landmarks drawn
// - draw_alignment_debug: Before/after alignment visualization

18
src/face_processing/mod.rs
View file

@ -1,18 +0,0 @@
//! Image processing module.
//!
//! This module handles face detection, landmark detection,
//! alignment, and image transformation.
mod crop;
pub mod debug;
mod orientation;
pub mod types;
pub use crop::*;
pub use orientation::load_image_with_orientation;
pub use types::*;
// TODO: Implement these modules
// mod landmarks;
// mod alignment;
// mod filters;

1
src/lib.rs
View file

@ -7,7 +7,6 @@
pub mod config;
pub mod error;
pub mod face_processing;
pub mod immich_api;
pub mod job;
pub mod models;

2
src/models/dlib_landmarks.rs
View file

@ -4,7 +4,7 @@
//! in a thread-safe singleton to avoid reloading the model for every image.
use crate::error::{Error, Result};
use crate::face_processing::types::{Landmarks, Point};
use crate::pipeline::{Landmarks, Point};
use dlib_face_recognition::{
FaceDetector, FaceDetectorTrait, ImageMatrix, LandmarkPredictor, LandmarkPredictorTrait,
Rectangle,

2
src/models/dmhead.rs
View file

@ -8,7 +8,7 @@
//! Output: [yaw, pitch, roll] in degrees
use crate::error::{Error, Result};
use crate::face_processing::types::HeadPose;
use crate::pipeline::HeadPose;
use image::DynamicImage;
use ndarray::Array4;
use ort::session::{builder::GraphOptimizationLevel, Session};

2
src/face_processing/crop.rs → src/pipeline/crop_utils.rs
View file

@ -3,8 +3,8 @@
//! Extracts and resizes face regions from images using bounding box data.
use crate::error::{Error, Result};
use crate::face_processing::types::BoundingBox;
use crate::immich_api::FaceData;
use crate::pipeline::BoundingBox;
use image::imageops::FilterType;
use image::{DynamicImage, GenericImageView};

32
src/pipeline/mod.rs
View file

@ -13,10 +13,16 @@
//! let result = pipeline.execute(ctx, &config, &cancel_token, &skip_stats).await;
//! ```
mod crop_utils;
mod orientation;
mod traits;
mod types;
pub mod steps;
pub use crop_utils::{crop_face_with_intermediate, CropResult};
pub use orientation::load_image_with_orientation;
pub use traits::*;
pub use types::*;
use crate::config::Config;
use crate::web::AtomicSkipStats;
@ -87,14 +93,18 @@ impl Pipeline {
/// Create the default processing pipeline with standard steps.
///
/// Pipeline order:
/// 1. FaceResolutionStep - Validate face size from Immich metadata
/// 2. DecodeImageStep - Load and orient the image
/// 3. BrightnessStep - Filter by luminance
/// 4. CropFaceStep - Extract face region with padding
/// 5. HeadPoseStep - Filter non-frontal faces (DMHead)
/// 6. LandmarksStep - Detect 68 facial landmarks (dlib)
/// 7. AlignmentStep - Align face based on eye positions
/// 8. ResizeStep - Final resize to output size
/// 1. FaceResolutionStep - Validate face size from Immich metadata (Validator)
/// 2. DecodeImageStep - Load and orient the image (Transform)
/// 3. BrightnessStep - Filter by luminance (Validator)
/// 4. CropFaceStep - Extract face region with padding (Transform)
/// 5. HeadPoseStep - Filter non-frontal faces (Validator)
/// 6. LandmarksStep - Detect 68 facial landmarks (Detector)
/// 7. EyeFilterStep - Filter closed eyes by EAR (Validator)
/// 8. AlignmentStep - Align face based on eye positions (Transform)
/// 9. ResizeStep - Final resize to output size (Transform)
///
/// Debug visualizations are only generated for validator steps:
/// BrightnessStep, HeadPoseStep, EyeFilterStep
pub fn with_default_steps() -> Self {
use steps::*;
@ -105,6 +115,7 @@ impl Pipeline {
pipeline.add_step(Box::new(CropFaceStep));
pipeline.add_step(Box::new(HeadPoseStep));
pipeline.add_step(Box::new(LandmarksStep));
pipeline.add_step(Box::new(EyeFilterStep));
pipeline.add_step(Box::new(AlignmentStep));
pipeline.add_step(Box::new(ResizeStep));
pipeline
@ -149,7 +160,7 @@ impl Pipeline {
// Generate debug visualization if enabled (step passed)
if config.processing.output.keep_intermediates {
if let Some(debug_img) = step.debug_visualize(&ctx) {
if let Some(debug_img) = step.debug_visualize(&ctx, config) {
ctx.add_debug_image(step.id(), debug_img, true);
}
}
@ -160,7 +171,7 @@ impl Pipeline {
// Generate debug visualization for the failing step if enabled
if config.processing.output.keep_intermediates {
if let Some(debug_img) = step.debug_visualize(&ctx) {
if let Some(debug_img) = step.debug_visualize(&ctx, config) {
ctx.add_debug_image(step.id(), debug_img, false);
}
}
@ -285,6 +296,7 @@ mod tests {
assert!(ids.contains(&"crop"));
assert!(ids.contains(&"head_pose"));
assert!(ids.contains(&"landmarks"));
assert!(ids.contains(&"eye_filter"));
assert!(ids.contains(&"alignment"));
assert!(ids.contains(&"resize"));
}

0
src/face_processing/orientation.rs → src/pipeline/orientation.rs
View file

68
src/pipeline/steps/alignment.rs
View file

@ -4,10 +4,9 @@
//! across all images in the timelapse.
use crate::config::Config;
use crate::face_processing::types::Point;
use crate::pipeline::{PipelineContext, ProcessingStep, StepOutcome};
use crate::pipeline::{Point, PipelineContext, ProcessingStep, StepOutcome};
use async_trait::async_trait;
use image::{DynamicImage, GenericImageView, Rgb, RgbImage};
use image::{DynamicImage, GenericImageView, Rgb};
use imageproc::geometric_transformations::{rotate_about_center, Interpolation};
/// Aligns faces based on eye positions.
@ -36,7 +35,7 @@ impl ProcessingStep for AlignmentStep {
}
// Get landmarks from previous step
let landmarks: crate::face_processing::types::Landmarks = match ctx
let landmarks: crate::pipeline::Landmarks = match ctx
.get_computed("landmarks")
.and_then(|v| v.as_landmarks())
{
@ -156,72 +155,13 @@ impl ProcessingStep for AlignmentStep {
StepOutcome::Continue(ctx)
}
fn debug_visualize(&self, ctx: &PipelineContext) -> Option<DynamicImage> {
// Get landmarks for visualization
let landmarks: &crate::face_processing::types::Landmarks =
ctx.get_computed("landmarks").and_then(|v| v.as_landmarks())?;
let image = ctx.image.as_ref()?;
let mut debug_img = image.to_rgb8();
let (width, height) = (debug_img.width(), debug_img.height());
// Draw target eye positions
let output_size = width; // Assuming square output
let eye_y = (output_size as f32 * 0.35) as u32; // Default eye_y_position
// Draw horizontal line at target eye Y position
for x in 0..width {
if eye_y < height {
debug_img.put_pixel(x, eye_y, Rgb([0, 255, 0]));
}
}
// Draw vertical lines at target eye X positions (assuming 0.3 inter_eye_distance)
let inter_eye = (output_size as f32 * 0.3) as u32;
let left_x = (width - inter_eye) / 2;
let right_x = left_x + inter_eye;
for y in 0..height {
if left_x < width {
debug_img.put_pixel(left_x, y, Rgb([0, 255, 0]));
}
if right_x < width {
debug_img.put_pixel(right_x, y, Rgb([0, 255, 0]));
}
}
// Draw actual eye positions
let left_eye = landmarks.left_eye_center();
let right_eye = landmarks.right_eye_center();
draw_marker(&mut debug_img, left_eye.x as u32, left_eye.y as u32, Rgb([255, 0, 0]));
draw_marker(&mut debug_img, right_eye.x as u32, right_eye.y as u32, Rgb([255, 0, 0]));
Some(DynamicImage::ImageRgb8(debug_img))
}
}
/// Draw a marker (small filled square) at the given position.
fn draw_marker(img: &mut RgbImage, x: u32, y: u32, color: Rgb<u8>) {
let (width, height) = (img.width(), img.height());
let size: i32 = 3;
for dy in 0..=size * 2 {
for dx in 0..=size * 2 {
let px = (x as i32 + dx - size) as u32;
let py = (y as i32 + dy - size) as u32;
if px < width && py < height {
img.put_pixel(px, py, color);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::immich_api::FaceData;
use image::RgbImage;
fn make_test_ctx() -> PipelineContext {
let face_data = FaceData {

2
src/pipeline/steps/brightness.rs
View file

@ -95,7 +95,7 @@ impl ProcessingStep for BrightnessStep {
StepOutcome::Continue(ctx)
}
fn debug_visualize(&self, ctx: &PipelineContext) -> Option<DynamicImage> {
fn debug_visualize(&self, ctx: &PipelineContext, _config: &Config) -> Option<DynamicImage> {
// Get brightness from computed values
let brightness = ctx
.get_computed("brightness")

28
src/pipeline/steps/crop.rs
View file

@ -3,11 +3,9 @@
//! Crops the face region from the full image using the bounding box data.
use crate::config::Config;
use crate::face_processing::crop_face_with_intermediate;
use crate::face_processing::debug::draw_crop_debug;
use crate::pipeline::crop_face_with_intermediate;
use crate::pipeline::{ComputedValue, PipelineContext, ProcessingStep, StepOutcome};
use async_trait::async_trait;
use image::DynamicImage;
/// Crops the face region from the full image.
///
@ -49,30 +47,6 @@ impl ProcessingStep for CropFaceStep {
}
}
fn debug_visualize(&self, ctx: &PipelineContext) -> Option<DynamicImage> {
// Draw the crop region on the original image
// Note: This requires access to the original image before cropping,
// which we don't have here. For now, we'll create the debug image
// during execution if needed. This is a limitation of the current
// design that could be addressed by storing the original image.
// For now, return None and handle debug visualization in the pipeline
// execution or via a separate mechanism
ctx.raw_bytes.as_ref()?;
// If we had the original image, we could do:
// Some(draw_crop_debug(&original, &ctx.face_data))
None
}
}
/// Generates a debug visualization of the crop region.
///
/// This can be called separately before the crop step to visualize
/// what will be cropped.
#[allow(dead_code)]
pub fn generate_crop_debug(image: &DynamicImage, ctx: &PipelineContext) -> DynamicImage {
draw_crop_debug(image, &ctx.face_data)
}
#[cfg(test)]

2
src/pipeline/steps/decode.rs
View file

@ -3,7 +3,7 @@
//! Decodes raw image bytes and applies EXIF orientation correction.
use crate::config::Config;
use crate::face_processing::load_image_with_orientation;
use crate::pipeline::load_image_with_orientation;
use crate::pipeline::{PipelineContext, ProcessingStep, StepOutcome};
use async_trait::async_trait;

296
src/pipeline/steps/eye_filter.rs Normal file
View file

@ -0,0 +1,296 @@
//! Eye aspect ratio (EAR) filter step.
//!
//! Filters images based on eye openness using the Eye Aspect Ratio computed
//! from facial landmarks.
use crate::config::Config;
use crate::pipeline::{Landmarks, PipelineContext, ProcessingStep, StepOutcome};
use async_trait::async_trait;
use image::{DynamicImage, Rgb, RgbImage};
/// Filters images where eyes appear closed based on Eye Aspect Ratio.
///
/// This validator step reads the EAR value computed by LandmarksStep
/// and skips images where the average EAR is below the configured threshold.
///
/// Must run after LandmarksStep.
pub struct EyeFilterStep;
#[async_trait]
impl ProcessingStep for EyeFilterStep {
fn id(&self) -> &'static str {
"eye_filter"
}
fn name(&self) -> &'static str {
"Eye Filter"
}
async fn execute(&self, ctx: PipelineContext, config: &Config) -> StepOutcome {
// Skip if eye filtering is disabled
if !config.processing.eye_filter.enabled {
return StepOutcome::Continue(ctx);
}
// Get EAR from computed values (set by LandmarksStep)
let avg_ear = match ctx.get_computed("ear").and_then(|v| v.as_float()) {
Some(ear) => ear,
None => {
// No EAR available - landmarks step must have been skipped
tracing::warn!("EAR not available for eye filter, skipping check");
return StepOutcome::Continue(ctx);
}
};
let min_ear = config.processing.eye_filter.min_ear;
if avg_ear < min_ear {
return StepOutcome::Skip {
ctx,
reason: "eyes_closed".to_string(),
detail: Some(format!("EAR {:.3} below threshold {:.3}", avg_ear, min_ear)),
};
}
tracing::trace!("Eye filter passed: EAR {:.3} >= {:.3}", avg_ear, min_ear);
StepOutcome::Continue(ctx)
}
fn debug_visualize(&self, ctx: &PipelineContext, _config: &Config) -> Option<DynamicImage> {
// Get landmarks for eye visualization
let landmarks: &Landmarks = ctx
.get_computed("landmarks")
.and_then(|v| v.as_landmarks())?;
// Get EAR values
let ear = landmarks.eye_aspect_ratio();
let avg_ear = (ear.left + ear.right) / 2.0;
// Get the current image to draw on
let image = ctx.image.as_ref()?;
let rgb = image.to_rgb8();
let (width, height) = (rgb.width(), rgb.height());
// Create a copy for visualization
let mut debug_img = rgb.clone();
// Draw eye landmarks (points 36-47)
let points = landmarks.points();
// Left eye (points 36-41) - yellow or red depending on EAR
let left_color = if ear.left >= 0.2 {
Rgb([0, 255, 0]) // Green - open
} else {
Rgb([255, 0, 0]) // Red - closed
};
for i in 36..42 {
let point = &points[i];
draw_cross(&mut debug_img, point.x as u32, point.y as u32, left_color);
}
// Right eye (points 42-47)
let right_color = if ear.right >= 0.2 {
Rgb([0, 255, 0]) // Green - open
} else {
Rgb([255, 0, 0]) // Red - closed
};
for i in 42..48 {
let point = &points[i];
draw_cross(&mut debug_img, point.x as u32, point.y as u32, right_color);
}
// Draw eye centers
let left_eye = landmarks.left_eye_center();
let right_eye = landmarks.right_eye_center();
draw_marker(&mut debug_img, left_eye.x as u32, left_eye.y as u32, Rgb([0, 255, 255]));
draw_marker(&mut debug_img, right_eye.x as u32, right_eye.y as u32, Rgb([0, 255, 255]));
// Draw info bar at bottom
let bar_height = 20u32;
let bar_y = height.saturating_sub(bar_height);
// Draw background
for y in bar_y..height {
for x in 0..width {
debug_img.put_pixel(x, y, Rgb([0, 0, 0]));
}
}
// Draw EAR values
let text = format!(
"L:{:.2} R:{:.2} Avg:{:.2}",
ear.left, ear.right, avg_ear
);
draw_simple_text(&mut debug_img, 5, bar_y + 6, &text, Rgb([255, 255, 255]));
Some(DynamicImage::ImageRgb8(debug_img))
}
}
/// Draw a small cross at the given position.
fn draw_cross(img: &mut RgbImage, x: u32, y: u32, color: Rgb<u8>) {
let (width, height) = (img.width(), img.height());
let size: i32 = 2;
// Check base coordinates are in bounds
if x >= width || y >= height {
return;
}
for dx in 0..=size * 2 {
let px = (x as i32 + dx - size) as u32;
if px < width && y < height {
img.put_pixel(px, y, color);
}
}
for dy in 0..=size * 2 {
let py = (y as i32 + dy - size) as u32;
if x < width && py < height {
img.put_pixel(x, py, color);
}
}
}
/// Draw a marker (small filled square) at the given position.
fn draw_marker(img: &mut RgbImage, x: u32, y: u32, color: Rgb<u8>) {
let (width, height) = (img.width(), img.height());
let size: i32 = 3;
for dy in 0..=size * 2 {
for dx in 0..=size * 2 {
let px = (x as i32 + dx - size) as u32;
let py = (y as i32 + dy - size) as u32;
if px < width && py < height {
img.put_pixel(px, py, color);
}
}
}
}
/// Draw simple text using a basic 5x7 pixel font.
fn draw_simple_text(img: &mut RgbImage, x: u32, y: u32, text: &str, color: Rgb<u8>) {
let (width, height) = (img.width(), img.height());
let mut cursor_x = x;
for ch in text.chars() {
let pattern = get_char_pattern(ch);
for (row_idx, row) in pattern.iter().enumerate() {
for col in 0..5 {
if (row >> (4 - col)) & 1 == 1 {
let px = cursor_x + col;
let py = y + row_idx as u32;
if px < width && py < height {
img.put_pixel(px, py, color);
}
}
}
}
cursor_x += 6; // 5 pixels wide + 1 pixel spacing
}
}
/// Get a 5x7 pixel pattern for a character.
fn get_char_pattern(ch: char) -> [u8; 7] {
match ch {
'0' => [0b01110, 0b10001, 0b10011, 0b10101, 0b11001, 0b10001, 0b01110],
'1' => [0b00100, 0b01100, 0b00100, 0b00100, 0b00100, 0b00100, 0b01110],
'2' => [0b01110, 0b10001, 0b00001, 0b00110, 0b01000, 0b10000, 0b11111],
'3' => [0b01110, 0b10001, 0b00001, 0b00110, 0b00001, 0b10001, 0b01110],
'4' => [0b00010, 0b00110, 0b01010, 0b10010, 0b11111, 0b00010, 0b00010],
'5' => [0b11111, 0b10000, 0b11110, 0b00001, 0b00001, 0b10001, 0b01110],
'6' => [0b00110, 0b01000, 0b10000, 0b11110, 0b10001, 0b10001, 0b01110],
'7' => [0b11111, 0b00001, 0b00010, 0b00100, 0b01000, 0b01000, 0b01000],
'8' => [0b01110, 0b10001, 0b10001, 0b01110, 0b10001, 0b10001, 0b01110],
'9' => [0b01110, 0b10001, 0b10001, 0b01111, 0b00001, 0b00010, 0b01100],
'L' => [0b10000, 0b10000, 0b10000, 0b10000, 0b10000, 0b10000, 0b11111],
'R' => [0b11110, 0b10001, 0b10001, 0b11110, 0b10100, 0b10010, 0b10001],
'A' => [0b01110, 0b10001, 0b10001, 0b11111, 0b10001, 0b10001, 0b10001],
'v' => [0b00000, 0b00000, 0b10001, 0b10001, 0b10001, 0b01010, 0b00100],
'g' => [0b00000, 0b00000, 0b01111, 0b10001, 0b01111, 0b00001, 0b01110],
':' => [0b00000, 0b00100, 0b00000, 0b00000, 0b00100, 0b00000, 0b00000],
'.' => [0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00100],
' ' => [0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00000],
_ => [0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00000, 0b00000],
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::immich_api::FaceData;
use crate::pipeline::ComputedValue;
fn make_test_ctx() -> PipelineContext {
let face_data = FaceData {
bounding_box_x1: 0.0,
bounding_box_y1: 0.0,
bounding_box_x2: 100.0,
bounding_box_y2: 100.0,
image_width: 100,
image_height: 100,
};
PipelineContext::new("test".to_string(), "2024-01-01".to_string(), face_data)
}
#[tokio::test]
async fn test_disabled_skips_check() {
let step = EyeFilterStep;
let mut ctx = make_test_ctx();
ctx.set_computed("ear", ComputedValue::Float(0.1)); // Below threshold
let mut config = Config::default();
config.processing.eye_filter.enabled = false;
match step.execute(ctx, &config).await {
StepOutcome::Continue(_) => {} // Expected
other => panic!("Expected Continue when disabled, got {:?}", other),
}
}
#[tokio::test]
async fn test_below_threshold_skips() {
let step = EyeFilterStep;
let mut ctx = make_test_ctx();
ctx.set_computed("ear", ComputedValue::Float(0.1)); // Below default 0.2 threshold
let mut config = Config::default();
config.processing.eye_filter.enabled = true;
config.processing.eye_filter.min_ear = 0.2;
match step.execute(ctx, &config).await {
StepOutcome::Skip { reason, .. } => {
assert_eq!(reason, "eyes_closed");
}
other => panic!("Expected Skip, got {:?}", other),
}
}
#[tokio::test]
async fn test_above_threshold_continues() {
let step = EyeFilterStep;
let mut ctx = make_test_ctx();
ctx.set_computed("ear", ComputedValue::Float(0.3)); // Above threshold
let mut config = Config::default();
config.processing.eye_filter.enabled = true;
config.processing.eye_filter.min_ear = 0.2;
match step.execute(ctx, &config).await {
StepOutcome::Continue(_) => {} // Expected
other => panic!("Expected Continue, got {:?}", other),
}
}
#[tokio::test]
async fn test_no_ear_continues() {
let step = EyeFilterStep;
let ctx = make_test_ctx(); // No EAR set
let mut config = Config::default();
config.processing.eye_filter.enabled = true;
match step.execute(ctx, &config).await {
StepOutcome::Continue(_) => {} // Expected - gracefully handles missing EAR
other => panic!("Expected Continue when no EAR, got {:?}", other),
}
}
}

2
src/pipeline/steps/head_pose.rs
View file

@ -136,7 +136,7 @@ impl ProcessingStep for HeadPoseStep {
StepOutcome::Continue(ctx)
}
fn debug_visualize(&self, ctx: &PipelineContext) -> Option<DynamicImage> {
fn debug_visualize(&self, ctx: &PipelineContext, _config: &Config) -> Option<DynamicImage> {
// Get head pose from computed values
let pose = ctx
.get_computed("head_pose")

97
src/pipeline/steps/landmarks.rs
View file

@ -3,20 +3,19 @@
//! Uses dlib to detect 68 facial landmarks for alignment and eye filtering.
use crate::config::Config;
use crate::face_processing::types::Landmarks;
use crate::models::DlibLandmarks;
use crate::pipeline::{ComputedValue, PipelineContext, ProcessingStep, StepOutcome};
use crate::pipeline::{ComputedValue, Landmarks, PipelineContext, ProcessingStep, StepOutcome};
use async_trait::async_trait;
use image::{DynamicImage, Rgb, RgbImage};
use tokio::task;
/// Detects facial landmarks and optionally filters based on eye aspect ratio.
/// Detects facial landmarks using dlib.
///
/// This step:
/// 1. Uses dlib to detect faces and 68 landmarks
/// 2. Stores Landmarks in ctx.computed["landmarks"]
/// 3. Computes EAR and stores in ctx.computed["ear"]
/// 4. Optionally skips if EAR is below threshold (eyes closed)
///
/// Eye filtering (skipping closed eyes) is handled by EyeFilterStep.
pub struct LandmarksStep;
#[async_trait]
@ -97,18 +96,6 @@ impl ProcessingStep for LandmarksStep {
// Store landmarks
ctx.set_computed("landmarks", ComputedValue::Landmarks(Box::new(landmarks)));
// Check eye filter if enabled
if config.processing.eye_filter.enabled {
let min_ear = config.processing.eye_filter.min_ear;
if avg_ear < min_ear {
return StepOutcome::Skip {
ctx,
reason: "eyes_closed".to_string(),
detail: Some(format!("EAR {:.3} below threshold {:.3}", avg_ear, min_ear)),
};
}
}
tracing::trace!(
"Landmarks detected: EAR left={:.3}, right={:.3}, avg={:.3}",
ear.left,
@ -118,87 +105,13 @@ impl ProcessingStep for LandmarksStep {
StepOutcome::Continue(ctx)
}
fn debug_visualize(&self, ctx: &PipelineContext) -> Option<DynamicImage> {
// Get landmarks from computed values
let landmarks: &Landmarks = ctx
.get_computed("landmarks")
.and_then(|v| v.as_landmarks())?;
// Get the current image to draw on
let image = ctx.image.as_ref()?;
let mut debug_img = image.to_rgb8();
// Draw all 68 landmark points
let points = landmarks.points();
for (i, point) in points.iter().enumerate() {
let x = point.x as u32;
let y = point.y as u32;
// Color-code different facial regions
let color = match i {
0..=16 => Rgb([255, 0, 0]), // Jaw (red)
17..=21 => Rgb([0, 255, 0]), // Left eyebrow (green)
22..=26 => Rgb([0, 255, 0]), // Right eyebrow (green)
27..=35 => Rgb([0, 0, 255]), // Nose (blue)
36..=41 => Rgb([255, 255, 0]), // Left eye (yellow)
42..=47 => Rgb([255, 255, 0]), // Right eye (yellow)
48..=67 => Rgb([255, 0, 255]), // Mouth (magenta)
_ => Rgb([255, 255, 255]), // Other (white)
};
// Draw a small cross at each point
draw_cross(&mut debug_img, x, y, color);
}
// Draw eye centers
let left_eye = landmarks.left_eye_center();
let right_eye = landmarks.right_eye_center();
draw_cross(
&mut debug_img,
left_eye.x as u32,
left_eye.y as u32,
Rgb([0, 255, 255]),
);
draw_cross(
&mut debug_img,
right_eye.x as u32,
right_eye.y as u32,
Rgb([0, 255, 255]),
);
Some(DynamicImage::ImageRgb8(debug_img))
}
}
/// Draw a small cross at the given position.
fn draw_cross(img: &mut RgbImage, x: u32, y: u32, color: Rgb<u8>) {
let (width, height) = (img.width(), img.height());
let size: i32 = 2;
// Check base coordinates are in bounds
if x >= width || y >= height {
return;
}
for dx in 0..=size * 2 {
let px = (x as i32 + dx - size) as u32;
if px < width && y < height {
img.put_pixel(px, y, color);
}
}
for dy in 0..=size * 2 {
let py = (y as i32 + dy - size) as u32;
if x < width && py < height {
img.put_pixel(x, py, color);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::immich_api::FaceData;
use image::{DynamicImage, RgbImage};
fn make_test_ctx() -> PipelineContext {
let face_data = FaceData {

2
src/pipeline/steps/mod.rs
View file

@ -7,6 +7,7 @@ mod alignment;
mod brightness;
mod crop;
mod decode;
mod eye_filter;
mod face_resolution;
mod head_pose;
mod landmarks;
@ -16,6 +17,7 @@ pub use alignment::AlignmentStep;
pub use brightness::BrightnessStep;
pub use crop::CropFaceStep;
pub use decode::DecodeImageStep;
pub use eye_filter::EyeFilterStep;
pub use face_resolution::FaceResolutionStep;
pub use head_pose::HeadPoseStep;
pub use landmarks::LandmarksStep;

6
src/pipeline/traits.rs
View file

@ -4,7 +4,7 @@
//! extensible image processing pipelines.
use crate::config::Config;
use crate::face_processing::types::{BoundingBox, HeadPose, Landmarks};
use crate::pipeline::types::{BoundingBox, HeadPose, Landmarks};
use crate::immich_api::FaceData;
use async_trait::async_trait;
use bytes::Bytes;
@ -243,7 +243,9 @@ pub trait ProcessingStep: Send + Sync {
///
/// Called after `execute()` if debug mode is enabled and the step
/// returned `Continue`. The returned image is saved to the debug folder.
fn debug_visualize(&self, _ctx: &PipelineContext) -> Option<DynamicImage> {
///
/// Steps should return `None` if they are disabled (check config.processing.X.enabled).
fn debug_visualize(&self, _ctx: &PipelineContext, _config: &Config) -> Option<DynamicImage> {
None
}
}

0
src/face_processing/types.rs → src/pipeline/types.rs
View file

44
src/web/handlers/config.rs
View file

@ -1,8 +1,8 @@
//! Configuration endpoints.
use crate::config::{
AlignmentConfig, BrightnessConfig, FaceResolutionConfig, OutputConfig, ProcessingConfig,
VideoConfig,
AlignmentConfig, BrightnessConfig, EyeFilterConfig, FaceResolutionConfig, HeadPoseConfig,
OutputConfig, ProcessingConfig, VideoConfig,
};
use crate::web::state::AppState;
use axum::{extract::State, http::StatusCode, response::Json};
@ -40,6 +40,8 @@ pub struct ProcessingConfigUpdate {
pub max_workers: Option<usize>,
pub face_resolution: Option<FaceResolutionConfig>,
pub brightness: Option<BrightnessConfig>,
pub head_pose: Option<HeadPoseConfig>,
pub eye_filter: Option<EyeFilterConfig>,
pub output: Option<OutputConfig>,
pub alignment: Option<AlignmentConfig>,
}
@ -117,6 +119,38 @@ fn validate_processing_config(proc: &ProcessingConfigUpdate) -> Result<(), Valid
}
}
if let Some(ref hp) = proc.head_pose {
if hp.enabled {
if hp.max_yaw < 0.0 || hp.max_yaw > 90.0 {
return Err(ValidationError::new(
"processing.head_pose.max_yaw",
format!("must be between 0 and 90, got {}", hp.max_yaw),
));
}
if hp.max_pitch < 0.0 || hp.max_pitch > 90.0 {
return Err(ValidationError::new(
"processing.head_pose.max_pitch",
format!("must be between 0 and 90, got {}", hp.max_pitch),
));
}
if hp.max_roll < 0.0 || hp.max_roll > 90.0 {
return Err(ValidationError::new(
"processing.head_pose.max_roll",
format!("must be between 0 and 90, got {}", hp.max_roll),
));
}
}
}
if let Some(ref ef) = proc.eye_filter {
if ef.enabled && !(0.0..=0.5).contains(&ef.min_ear) {
return Err(ValidationError::new(
"processing.eye_filter.min_ear",
format!("must be between 0.0 and 0.5, got {}", ef.min_ear),
));
}
}
if let Some(ref out) = proc.output {
if out.size < 64 {
return Err(ValidationError::new(
@ -211,6 +245,12 @@ pub async fn update_config(
if let Some(v) = proc.brightness {
config.processing.brightness = v;
}
if let Some(v) = proc.head_pose {
config.processing.head_pose = v;
}
if let Some(v) = proc.eye_filter {
config.processing.eye_filter = v;
}
if let Some(v) = proc.output {
config.processing.output = v;
}