danvics/immich-automated-selfie-timelapse
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Wip blurry image detection

Browse source
This commit is contained in:
Arnaud_Cayrol 2026-02-09 19:01:49 +01:00
parent 9d313382eb
commit 8735f8ef6b
10 changed files with 429 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -54,6 +54,7 @@ unicode-normalization = "0.1"
[dev-dependencies]
tokio-test = "0.4"
rand = "0.8"
[[bin]]
name = "immich-timelapse"

40
src/config.rs
View file

@ -119,6 +119,40 @@ impl FaceResolutionConfig {
}
}
/// Blur detection configuration.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BlurConfig {
/// Whether blur detection is enabled.
pub enabled: bool,
/// Minimum gradient magnitude threshold.
/// Images with gradient magnitude below this are considered blurry.
/// Uses Sobel operator for robust edge detection.
/// Typical values: 10-20 for strict filtering, 5-10 for moderate filtering.
pub min_sharpness: f32,
}
impl Default for BlurConfig {
fn default() -> Self {
Self {
enabled: false,
min_sharpness: 15.0,
}
}
}
impl BlurConfig {
/// Validate the configuration values.
pub fn validate(&self) -> Result<()> {
if self.enabled && self.min_sharpness < 0.0 {
return Err(Error::Config(
"Blur min_sharpness must be non-negative".to_string(),
));
}
Ok(())
}
}
/// Brightness validation configuration.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BrightnessConfig {
@ -377,6 +411,10 @@ pub struct ProcessingConfig {
#[serde(default)]
pub face_resolution: FaceResolutionConfig,
/// Blur detection settings.
#[serde(default)]
pub blur: BlurConfig,
/// Brightness validation settings.
#[serde(default)]
pub brightness: BrightnessConfig,
@ -403,6 +441,7 @@ impl Default for ProcessingConfig {
Self {
max_workers: num_cpus(),
face_resolution: FaceResolutionConfig::default(),
blur: BlurConfig::default(),
brightness: BrightnessConfig::default(),
head_pose: HeadPoseConfig::default(),
eye_filter: EyeFilterConfig::default(),
@ -416,6 +455,7 @@ impl ProcessingConfig {
/// Validate all step configurations.
pub fn validate(&self) -> Result<()> {
self.face_resolution.validate()?;
self.blur.validate()?;
self.brightness.validate()?;
self.head_pose.validate()?;
self.eye_filter.validate()?;

19
src/pipeline/mod.rs
View file

@ -100,10 +100,10 @@ impl Pipeline {
/// 2. DecodeImageStep - Load and orient the image (always)
/// 3. CropAndResizeStep - Extract face region with padding and resize to output size (always)
/// 4. BrightnessStep - Filter by luminance on face region within cropped image (if enabled)
/// 5. HeadPoseStep - Filter non-frontal faces (if enabled)
/// 6. LandmarksStep - Detect 68 facial landmarks (always)
/// 7. EyeFilterStep - Filter closed eyes by EAR (if enabled)
/// 8. AlignmentStep - Align face based on eye positions and resize to final output (always)
/// 5. BlurStep - Filter blurry images using gradient magnitude (if enabled) /// 6. HeadPoseStep - Filter non-frontal faces (if enabled)
/// 7. LandmarksStep - Detect 68 facial landmarks (always)
/// 8. EyeFilterStep - Filter closed eyes by EAR (if enabled)
/// 9. AlignmentStep - Align face based on eye positions and resize to final output (always)
pub fn with_steps_from_config(config: &Config) -> Self {
use steps::*;
@ -125,6 +125,11 @@ impl Pipeline {
pipeline.add_step(Box::new(BrightnessStep));
}
// Optional: Blur detection
if config.processing.blur.enabled {
pipeline.add_step(Box::new(BlurStep));
}
// Optional: Head pose validation
if config.processing.head_pose.enabled {
pipeline.add_step(Box::new(HeadPoseStep));
@ -154,6 +159,7 @@ impl Pipeline {
pipeline.add_step(Box::new(DecodeImageStep));
pipeline.add_step(Box::new(CropAndResizeStep));
pipeline.add_step(Box::new(BrightnessStep));
pipeline.add_step(Box::new(BlurStep));
pipeline.add_step(Box::new(HeadPoseStep));
pipeline.add_step(Box::new(LandmarksStep));
pipeline.add_step(Box::new(EyeFilterStep));
@ -353,8 +359,9 @@ mod tests {
assert!(ids.contains(&"face_resolution"));
assert!(ids.contains(&"decode"));
assert!(ids.contains(&"brightness"));
assert!(ids.contains(&"crop_and_resize"));
assert!(ids.contains(&"brightness"));
assert!(ids.contains(&"blur"));
assert!(ids.contains(&"head_pose"));
assert!(ids.contains(&"landmarks"));
assert!(ids.contains(&"eye_filter"));
@ -366,6 +373,7 @@ mod tests {
// Config with all optional steps disabled
let mut config = Config::default();
config.processing.face_resolution.enabled = false;
config.processing.blur.enabled = false;
config.processing.brightness.enabled = false;
config.processing.head_pose.enabled = false;
config.processing.eye_filter.enabled = false;
@ -385,6 +393,7 @@ mod tests {
fn test_pipeline_from_config_with_eye_filter() {
let mut config = Config::default();
config.processing.face_resolution.enabled = false;
config.processing.blur.enabled = false;
config.processing.brightness.enabled = false;
config.processing.head_pose.enabled = false;
config.processing.eye_filter.enabled = true;

370
src/pipeline/steps/blur.rs Normal file
View file

@ -0,0 +1,370 @@
//! Blur detection step.
//!
//! Detects blurry faces using gradient magnitude analysis within the face region:
//! 1. Convert image to grayscale
//! 2. Apply Sobel operator (Sobel-X and Sobel-Y) within the face bounding box
//! 3. Compute gradient magnitude for each pixel: sqrt(gx² + gy²)
//! 4. Calculate mean gradient magnitude
//! 5. Low gradient magnitude → blurry face (weak, spread-out edges)
use crate::config::Config;
use crate::pipeline::{
computed_keys, draw_simple_text, ComputedValue, PipelineContext, ProcessingStep, StepOutcome,
};
use async_trait::async_trait;
use image::{DynamicImage, Rgb};
pub struct BlurStep;
impl BlurStep {
/// Calculate the mean gradient magnitude of a specific region within an image.
///
/// This method:
/// 1. Converts the image to grayscale
/// 2. Applies Sobel-X and Sobel-Y kernels within the specified region:
/// Sobel-X: [-1 0 1] Sobel-Y: [-1 -2 -1]
/// [-2 0 2] [ 0 0 0]
/// [-1 0 1] [ 1 2 1]
/// 3. Computes gradient magnitude for each pixel: sqrt(gx² + gy²)
/// 4. Returns the mean gradient magnitude
///
/// Higher gradient magnitude = sharper image (strong, concentrated edges)
/// Lower gradient magnitude = blurrier image (weak, spread-out edges)
///
/// # Arguments
/// * `image` - The full image
/// * `x1`, `y1`, `x2`, `y2` - Bounding box coordinates (pixels, clamped to image bounds)
fn calculate_gradient_magnitude_in_region(
image: &DynamicImage,
x1: u32,
y1: u32,
x2: u32,
y2: u32,
) -> f32 {
let gray = image.to_luma8();
let (img_width, img_height) = gray.dimensions();
// Clamp coordinates to image bounds
let x1 = x1.min(img_width.saturating_sub(1));
let y1 = y1.min(img_height.saturating_sub(1));
let x2 = x2.min(img_width);
let y2 = y2.min(img_height);
// Ensure valid region with space for 3x3 kernel
if x2 <= x1 + 2 || y2 <= y1 + 2 {
return 0.0;
}
// Apply Sobel filter within the region and compute gradient magnitudes
let mut sum_magnitude = 0.0f32;
let mut count = 0usize;
for y in (y1 + 1)..(y2 - 1) {
for x in (x1 + 1)..(x2 - 1) {
// Get the 3x3 neighborhood
let p00 = gray.get_pixel(x - 1, y - 1)[0] as i32;
let p01 = gray.get_pixel(x, y - 1)[0] as i32;
let p02 = gray.get_pixel(x + 1, y - 1)[0] as i32;
let p10 = gray.get_pixel(x - 1, y)[0] as i32;
let p12 = gray.get_pixel(x + 1, y)[0] as i32;
let p20 = gray.get_pixel(x - 1, y + 1)[0] as i32;
let p21 = gray.get_pixel(x, y + 1)[0] as i32;
let p22 = gray.get_pixel(x + 1, y + 1)[0] as i32;
// Apply Sobel-X kernel: [-1 0 1; -2 0 2; -1 0 1]
let gx = -p00 + p02 - 2 * p10 + 2 * p12 - p20 + p22;
// Apply Sobel-Y kernel: [-1 -2 -1; 0 0 0; 1 2 1]
let gy = -p00 - 2 * p01 - p02 + p20 + 2 * p21 + p22;
// Compute gradient magnitude: sqrt(gx² + gy²)
let magnitude = ((gx * gx + gy * gy) as f32).sqrt();
sum_magnitude += magnitude;
count += 1;
}
}
if count == 0 {
return 0.0;
}
// Return mean gradient magnitude
sum_magnitude / count as f32
}
}
#[async_trait]
impl ProcessingStep for BlurStep {
fn id(&self) -> &'static str {
"blur"
}
fn name(&self) -> &'static str {
"Blur Detection"
}
fn provides(&self) -> Vec<&'static str> {
vec![computed_keys::BLUR_METRIC]
}
async fn execute(&self, mut ctx: PipelineContext, config: &Config) -> StepOutcome {
let step_config = &config.processing.blur;
let image = match ctx.require_image("blur detection") {
Ok(img) => img,
Err(e) => return StepOutcome::Error { ctx, error: e },
};
let img_width = image.width();
let img_height = image.height();
// Use FACE_RECT if available (face region within the cropped image),
// otherwise analyze the entire cropped image
let (x1, y1, x2, y2) = if let Some(face_rect) = ctx
.get_computed(computed_keys::FACE_RECT)
.and_then(|v| v.as_face_rect())
{
// Use the face rectangle within the cropped image
let x1 = (face_rect.x1.max(0.0) as u32).min(img_width.saturating_sub(1));
let y1 = (face_rect.y1.max(0.0) as u32).min(img_height.saturating_sub(1));
let x2 = (face_rect.x2.max(0.0) as u32).min(img_width);
let y2 = (face_rect.y2.max(0.0) as u32).min(img_height);
(x1, y1, x2, y2)
} else {
// No face rect available, use the entire cropped image
(0, 0, img_width, img_height)
};
let gradient_magnitude =
Self::calculate_gradient_magnitude_in_region(image, x1, y1, x2, y2);
// Store computed gradient magnitude for potential use by other steps
ctx.set_computed(
computed_keys::BLUR_METRIC,
ComputedValue::Float(gradient_magnitude),
);
if gradient_magnitude < step_config.min_sharpness {
return StepOutcome::Skip {
ctx,
reason: "too_blurry".to_string(),
detail: Some(format!(
"gradient: {:.1} (min: {:.1})",
gradient_magnitude, step_config.min_sharpness
)),
};
}
StepOutcome::Continue(ctx)
}
fn debug_visualize(&self, ctx: &PipelineContext, _config: &Config) -> Option<DynamicImage> {
// Get blur metric (gradient magnitude) from computed values
let gradient_mag = ctx
.get_computed(computed_keys::BLUR_METRIC)
.and_then(|v| v.as_float())?;
// Get the current image to draw on
let image = ctx.image.as_ref()?;
let rgb = image.to_rgb8();
let (width, height) = (rgb.width(), rgb.height());
// Create a copy for visualization
let mut debug_img = rgb.clone();
// Draw the face bounding box (FACE_RECT if available, otherwise full image)
let (x1, y1, x2, y2) = if let Some(face_rect) = ctx
.get_computed(computed_keys::FACE_RECT)
.and_then(|v| v.as_face_rect())
{
let x1 = (face_rect.x1.max(0.0) as u32).min(width.saturating_sub(1));
let y1 = (face_rect.y1.max(0.0) as u32).min(height.saturating_sub(1));
let x2 = (face_rect.x2.max(0.0) as u32).min(width);
let y2 = (face_rect.y2.max(0.0) as u32).min(height);
(x1, y1, x2, y2)
} else {
// No face rect, show that we analyzed the full cropped image
(0, 0, width, height)
};
// Draw rectangle outline (cyan color for visibility)
let rect_color = Rgb([0, 255, 255]);
let thickness = 2;
// Draw horizontal lines (top and bottom)
for t in 0..thickness {
for x in x1..x2 {
if y1 + t < height {
debug_img.put_pixel(x, y1 + t, rect_color);
}
if y2 > t && y2 - t - 1 < height {
debug_img.put_pixel(x, y2 - t - 1, rect_color);
}
}
}
// Draw vertical lines (left and right)
for t in 0..thickness {
for y in y1..y2 {
if x1 + t < width {
debug_img.put_pixel(x1 + t, y, rect_color);
}
if x2 > t && x2 - t - 1 < width {
debug_img.put_pixel(x2 - t - 1, y, rect_color);
}
}
}
// Draw a horizontal gradient magnitude bar at the bottom
let bar_height = 20u32;
let bar_y = height.saturating_sub(bar_height);
let bar_width = (width as f32 * 0.8) as u32;
let bar_x = (width - bar_width) / 2;
// Draw background (dark gray)
for y in bar_y..height {
for x in 0..width {
debug_img.put_pixel(x, y, Rgb([40, 40, 40]));
}
}
// Draw bar outline (white)
let outline_y = bar_y + 4;
let outline_height = bar_height - 8;
for x in bar_x..bar_x + bar_width {
debug_img.put_pixel(x, outline_y, Rgb([200, 200, 200]));
debug_img.put_pixel(x, outline_y + outline_height - 1, Rgb([200, 200, 200]));
}
for y in outline_y..outline_y + outline_height {
debug_img.put_pixel(bar_x, y, Rgb([200, 200, 200]));
debug_img.put_pixel(bar_x + bar_width - 1, y, Rgb([200, 200, 200]));
}
// Fill the bar based on gradient magnitude value (scale: 0-50 maps to 0-100% bar)
let max_gradient = 50.0;
let normalized = (gradient_mag / max_gradient).clamp(0.0, 1.0);
let fill_width = ((bar_width - 4) as f32 * normalized) as u32;
let fill_color = gradient_to_color(gradient_mag);
for y in (outline_y + 2)..(outline_y + outline_height - 2) {
for x in (bar_x + 2)..(bar_x + 2 + fill_width) {
if x < width {
debug_img.put_pixel(x, y, fill_color);
}
}
}
// Draw gradient magnitude text value
let text = format!("Grad:{:.1}", gradient_mag);
draw_simple_text(&mut debug_img, 5, bar_y + 6, &text, Rgb([255, 255, 255]));
Some(DynamicImage::ImageRgb8(debug_img))
}
}
/// Convert gradient magnitude value to a color (red for blurry, green for sharp)
fn gradient_to_color(gradient_mag: f32) -> Rgb<u8> {
// Very low gradient (< 10) = red (blurry)
// Medium gradient (10-20) = yellow/orange
// High gradient (> 20) = green (sharp)
if gradient_mag < 10.0 {
Rgb([255, 80, 80]) // Red
} else if gradient_mag < 20.0 {
Rgb([255, 200, 80]) // Yellow/orange
} else {
Rgb([80, 255, 80]) // Green
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::immich_api::FaceData;
use image::{DynamicImage, Rgb, RgbImage};
fn make_ctx_with_image(image: DynamicImage) -> PipelineContext {
let face_data = FaceData {
bounding_box_x1: 0.0,
bounding_box_y1: 0.0,
bounding_box_x2: 100.0,
bounding_box_y2: 100.0,
image_width: 100,
image_height: 100,
};
PipelineContext::new("test".to_string(), "2024-01-01".to_string(), face_data)
.with_image(image)
}
fn create_solid_image(r: u8, g: u8, b: u8) -> DynamicImage {
let img = RgbImage::from_fn(100, 100, |_, _| Rgb([r, g, b]));
DynamicImage::ImageRgb8(img)
}
fn create_checkerboard_image() -> DynamicImage {
// Create a sharp checkerboard pattern (high variance)
let img = RgbImage::from_fn(100, 100, |x, y| {
if (x / 10 + y / 10) % 2 == 0 {
Rgb([255, 255, 255])
} else {
Rgb([0, 0, 0])
}
});
DynamicImage::ImageRgb8(img)
}
#[test]
fn test_calculate_gradient_magnitude_uniform() {
// Uniform image should have very low gradient magnitude (no edges)
let img = create_solid_image(128, 128, 128);
let gradient_mag = BlurStep::calculate_gradient_magnitude_in_region(&img, 0, 0, 100, 100);
assert!(
gradient_mag < 1.0,
"Uniform image should have near-zero gradient magnitude"
);
}
#[test]
fn test_calculate_gradient_magnitude_sharp() {
// Checkerboard should have high gradient magnitude (many edges)
let img = create_checkerboard_image();
let gradient_mag = BlurStep::calculate_gradient_magnitude_in_region(&img, 0, 0, 100, 100);
assert!(
gradient_mag > 15.0,
"Sharp checkerboard should have high gradient magnitude, got {}",
gradient_mag
);
}
#[tokio::test]
async fn test_blur_too_blurry() {
let step = BlurStep;
let img = create_solid_image(128, 128, 128); // Very low gradient magnitude
let ctx = make_ctx_with_image(img);
let mut config = Config::default();
config.processing.blur.enabled = true;
config.processing.blur.min_sharpness = 15.0;
match step.execute(ctx, &config).await {
StepOutcome::Skip { reason, .. } => {
assert_eq!(reason, "too_blurry");
}
_ => panic!("Expected Skip for blurry image"),
}
}
#[tokio::test]
async fn test_blur_sharp_image() {
let step = BlurStep;
let img = create_checkerboard_image(); // High gradient magnitude
let ctx = make_ctx_with_image(img);
let mut config = Config::default();
config.processing.blur.enabled = true;
config.processing.blur.min_sharpness = 15.0;
match step.execute(ctx, &config).await {
StepOutcome::Continue(_) => {} // Should pass
_ => panic!("Expected Continue for sharp image"),
}
}
}

25
src/pipeline/steps/brightness.rs
View file

@ -118,11 +118,6 @@ impl ProcessingStep for BrightnessStep {
// Store computed brightness for potential use by other steps
ctx.set_computed(computed_keys::BRIGHTNESS, ComputedValue::Float(brightness));
// Skip validation if disabled
if !step_config.enabled {
return StepOutcome::Continue(ctx);
}
if brightness < step_config.min_brightness {
return StepOutcome::Skip {
ctx,
@ -307,26 +302,6 @@ mod tests {
assert!(brightness > 0.45 && brightness < 0.55);
}
#[tokio::test]
async fn test_brightness_disabled() {
let step = BrightnessStep;
let img = create_solid_image(128, 128, 128);
let ctx = make_ctx_with_image(img);
let config = Config::default(); // brightness.enabled = false by default
match step.execute(ctx, &config).await {
StepOutcome::Continue(new_ctx) => {
// Should still compute brightness even when disabled
let brightness = new_ctx
.get_computed(computed_keys::BRIGHTNESS)
.and_then(|v| v.as_float())
.unwrap();
assert!(brightness > 0.45 && brightness < 0.55);
}
_ => panic!("Expected Continue"),
}
}
#[tokio::test]
async fn test_brightness_too_dark() {
let step = BrightnessStep;

19
src/pipeline/steps/eye_filter.rs
View file

@ -29,11 +29,6 @@ impl ProcessingStep for EyeFilterStep {
}
async fn execute(&self, ctx: PipelineContext, config: &Config) -> StepOutcome {
// Skip if eye filtering is disabled
if !config.processing.eye_filter.enabled {
return StepOutcome::Continue(ctx);
}
// Get EAR from computed values (set by LandmarksStep)
let avg_ear = match ctx
.get_computed(computed_keys::EAR)
@ -198,20 +193,6 @@ mod tests {
PipelineContext::new("test".to_string(), "2024-01-01".to_string(), face_data)
}
#[tokio::test]
async fn test_disabled_skips_check() {
let step = EyeFilterStep;
let mut ctx = make_test_ctx();
ctx.set_computed(computed_keys::EAR, ComputedValue::Float(0.1)); // Below threshold
let mut config = Config::default();
config.processing.eye_filter.enabled = false;
match step.execute(ctx, &config).await {
StepOutcome::Continue(_) => {} // Expected
other => panic!("Expected Continue when disabled, got {:?}", other),
}
}
#[tokio::test]
async fn test_below_threshold_skips() {
let step = EyeFilterStep;

19
src/pipeline/steps/face_resolution.rs
View file

@ -24,12 +24,6 @@ impl ProcessingStep for FaceResolutionStep {
async fn execute(&self, mut ctx: PipelineContext, config: &Config) -> StepOutcome {
let step_config = &config.processing.face_resolution;
// Skip this step if disabled
if !step_config.enabled {
return StepOutcome::Continue(ctx);
}
let face_data = &ctx.face_data;
// Calculate face size in pixels from bounding box
@ -106,17 +100,4 @@ mod tests {
}
}
#[tokio::test]
async fn test_step_disabled() {
let step = FaceResolutionStep;
let ctx = make_ctx(50.0); // Would normally be too small
let mut config = Config::default();
config.processing.face_resolution.enabled = false;
match step.execute(ctx, &config).await {
StepOutcome::Continue(_) => {} // Should pass through when disabled
_ => panic!("Expected Continue when disabled"),
}
}
}

23
src/pipeline/steps/head_pose.rs
View file

@ -171,11 +171,6 @@ impl ProcessingStep for HeadPoseStep {
}
async fn execute(&self, mut ctx: PipelineContext, config: &Config) -> StepOutcome {
// Skip if head pose filtering is disabled
if !config.processing.head_pose.enabled {
return StepOutcome::Continue(ctx);
}
let image: &DynamicImage = match ctx.require_image("head pose estimation") {
Ok(img) => img,
Err(e) => return StepOutcome::Error { ctx, error: e },
@ -366,7 +361,6 @@ impl ProcessingStep for HeadPoseStep {
mod tests {
use super::*;
use crate::immich_api::FaceData;
use image::RgbImage;
fn make_test_ctx() -> PipelineContext {
let face_data = FaceData {
@ -380,23 +374,6 @@ mod tests {
PipelineContext::new("test".to_string(), "2024-01-01".to_string(), face_data)
}
#[tokio::test]
async fn test_disabled_skips_check() {
let step = HeadPoseStep;
let ctx = make_test_ctx();
let mut config = Config::default();
config.processing.head_pose.enabled = false;
// Create a dummy image
let img = DynamicImage::ImageRgb8(RgbImage::new(100, 100));
let ctx = ctx.with_image(img);
match step.execute(ctx, &config).await {
StepOutcome::Continue(_) => {} // Expected
other => panic!("Expected Continue when disabled, got {:?}", other),
}
}
#[tokio::test]
async fn test_no_image_error() {
let step = HeadPoseStep;

2
src/pipeline/steps/mod.rs
View file

@ -4,6 +4,7 @@
//! operation in the image processing pipeline.
mod alignment;
mod blur;
mod brightness;
mod crop_and_resize;
mod decode;
@ -13,6 +14,7 @@ mod head_pose;
mod landmarks;
pub use alignment::AlignmentStep;
pub use blur::BlurStep;
pub use brightness::BrightnessStep;
pub use crop_and_resize::CropAndResizeStep;
pub use decode::DecodeImageStep;

2
src/pipeline/traits.rs
View file

@ -17,6 +17,8 @@ use std::fmt;
pub mod computed_keys {
/// Brightness value (0.0 - 1.0) computed by BrightnessStep.
pub const BRIGHTNESS: &str = "brightness";
/// Blur metric (gradient magnitude) computed by BlurStep.
pub const BLUR_METRIC: &str = "blur_metric";
/// Face size in pixels computed by FaceResolutionStep.
pub const FACE_SIZE: &str = "face_size";
/// Eye Aspect Ratio computed by LandmarksStep.