Add date proximity limitting
This commit is contained in:
parent
fb4fc59e84
commit
0ae09da074
7 changed files with 228 additions and 9 deletions
|
|
@ -78,6 +78,11 @@
|
||||||
month: config.processing.timestamp.month,
|
month: config.processing.timestamp.month,
|
||||||
day: config.processing.timestamp.day,
|
day: config.processing.timestamp.day,
|
||||||
},
|
},
|
||||||
|
photo_limit: {
|
||||||
|
enabled: config.processing.photo_limit.enabled,
|
||||||
|
max_photos: Number(config.processing.photo_limit.max_photos),
|
||||||
|
time_range: config.processing.photo_limit.time_range,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
video: {
|
video: {
|
||||||
enabled: config.video.enabled,
|
enabled: config.video.enabled,
|
||||||
|
|
@ -408,6 +413,49 @@
|
||||||
<input id="keep-intermediates" type="checkbox" bind:checked={config.processing.output.keep_intermediates} />
|
<input id="keep-intermediates" type="checkbox" bind:checked={config.processing.output.keep_intermediates} />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Photo Limit Section -->
|
||||||
|
<div class="setting-section">
|
||||||
|
<div class="section-header">
|
||||||
|
<span class="section-title">Photo Limit</span>
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
bind:checked={config.processing.photo_limit.enabled}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if config.processing.photo_limit.enabled}
|
||||||
|
<div class="setting-row sub-setting">
|
||||||
|
<label for="max-photos">
|
||||||
|
<span class="setting-label">Max Photos</span>
|
||||||
|
<span class="setting-hint">Maximum photos per time range</span>
|
||||||
|
</label>
|
||||||
|
<div class="setting-control">
|
||||||
|
<input
|
||||||
|
id="max-photos"
|
||||||
|
type="range"
|
||||||
|
bind:value={config.processing.photo_limit.max_photos}
|
||||||
|
min="1"
|
||||||
|
max="10"
|
||||||
|
step="1"
|
||||||
|
/>
|
||||||
|
<span class="value">{config.processing.photo_limit.max_photos}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="setting-row sub-setting">
|
||||||
|
<label for="time-range">
|
||||||
|
<span class="setting-label">Time Range</span>
|
||||||
|
<span class="setting-hint">Group photos by this period</span>
|
||||||
|
</label>
|
||||||
|
<select id="time-range" bind:value={config.processing.photo_limit.time_range}>
|
||||||
|
<option value="day">Day</option>
|
||||||
|
<option value="week">Week</option>
|
||||||
|
<option value="month">Month</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Timestamp Section -->
|
<!-- Timestamp Section -->
|
||||||
<div class="setting-section">
|
<div class="setting-section">
|
||||||
<div class="section-header">
|
<div class="section-header">
|
||||||
|
|
|
||||||
|
|
@ -166,6 +166,11 @@ export const DEFAULT_CONFIG = {
|
||||||
month: false,
|
month: false,
|
||||||
day: false,
|
day: false,
|
||||||
},
|
},
|
||||||
|
photo_limit: {
|
||||||
|
enabled: false,
|
||||||
|
max_photos: 1,
|
||||||
|
time_range: 'day',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
video: {
|
video: {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
|
|
|
||||||
|
|
@ -445,6 +445,59 @@ impl TimestampConfig {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Time range granularity for photo limiting.
|
||||||
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum TimeRange {
|
||||||
|
#[default]
|
||||||
|
Day,
|
||||||
|
Week,
|
||||||
|
Month,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Photo limit configuration.
|
||||||
|
///
|
||||||
|
/// Limits the number of successfully processed photos per time range
|
||||||
|
/// (day/week/month) to avoid over-representation of busy periods.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct PhotoLimitConfig {
|
||||||
|
/// Whether photo limiting is enabled.
|
||||||
|
pub enabled: bool,
|
||||||
|
|
||||||
|
/// Maximum number of photos per time range.
|
||||||
|
pub max_photos: u32,
|
||||||
|
|
||||||
|
/// Time range granularity.
|
||||||
|
pub time_range: TimeRange,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for PhotoLimitConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
enabled: false,
|
||||||
|
max_photos: 1,
|
||||||
|
time_range: TimeRange::Day,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PhotoLimitConfig {
|
||||||
|
/// Validate the configuration values.
|
||||||
|
pub fn validate(&self) -> Result<()> {
|
||||||
|
if self.enabled && self.max_photos == 0 {
|
||||||
|
return Err(Error::Config(
|
||||||
|
"Photo limit max_photos must be greater than 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if self.enabled && self.max_photos > 100 {
|
||||||
|
return Err(Error::Config(
|
||||||
|
"Photo limit max_photos must be at most 100".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Main Processing Configuration
|
// Main Processing Configuration
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
@ -490,6 +543,10 @@ pub struct ProcessingConfig {
|
||||||
/// Timestamp overlay settings.
|
/// Timestamp overlay settings.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub timestamp: TimestampConfig,
|
pub timestamp: TimestampConfig,
|
||||||
|
|
||||||
|
/// Photo limit settings.
|
||||||
|
#[serde(default)]
|
||||||
|
pub photo_limit: PhotoLimitConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for ProcessingConfig {
|
impl Default for ProcessingConfig {
|
||||||
|
|
@ -504,6 +561,7 @@ impl Default for ProcessingConfig {
|
||||||
output: OutputConfig::default(),
|
output: OutputConfig::default(),
|
||||||
alignment: AlignmentConfig::default(),
|
alignment: AlignmentConfig::default(),
|
||||||
timestamp: TimestampConfig::default(),
|
timestamp: TimestampConfig::default(),
|
||||||
|
photo_limit: PhotoLimitConfig::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -519,6 +577,7 @@ impl ProcessingConfig {
|
||||||
self.output.validate()?;
|
self.output.validate()?;
|
||||||
self.alignment.validate()?;
|
self.alignment.validate()?;
|
||||||
self.timestamp.validate()?;
|
self.timestamp.validate()?;
|
||||||
|
self.photo_limit.validate()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@
|
||||||
|
|
||||||
mod processing;
|
mod processing;
|
||||||
|
|
||||||
|
use crate::config::{PhotoLimitConfig, TimeRange};
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
use crate::immich_api::{Asset, FaceData, ImmichClient};
|
use crate::immich_api::{Asset, FaceData, ImmichClient};
|
||||||
use crate::pipeline::Pipeline;
|
use crate::pipeline::Pipeline;
|
||||||
|
|
@ -17,12 +18,85 @@ use crate::web::{AppState, AtomicSkipStats, JobStatus, Progress, SkipStats};
|
||||||
|
|
||||||
use processing::{process_single_asset, AssetProcessResult, DebugDirs, OutputDirs};
|
use processing::{process_single_asset, AssetProcessResult, DebugDirs, OutputDirs};
|
||||||
|
|
||||||
|
use chrono::NaiveDate;
|
||||||
|
use std::collections::HashMap;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::atomic::{AtomicU32, Ordering};
|
use std::sync::atomic::{AtomicU32, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::{Arc, RwLock};
|
||||||
use tokio::sync::Semaphore;
|
use tokio::sync::Semaphore;
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
|
/// Tracks per-time-bucket photo counts for limiting.
|
||||||
|
///
|
||||||
|
/// Thread-safe: uses atomic counters with fetch_add + undo pattern
|
||||||
|
/// so concurrent workers can claim slots without races.
|
||||||
|
pub struct PhotoLimitTracker {
|
||||||
|
max_photos: u32,
|
||||||
|
time_range: TimeRange,
|
||||||
|
buckets: RwLock<HashMap<String, Arc<AtomicU32>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PhotoLimitTracker {
|
||||||
|
/// Create a tracker if photo limiting is enabled, otherwise return None.
|
||||||
|
pub fn new(config: &PhotoLimitConfig) -> Option<Self> {
|
||||||
|
if !config.enabled {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(Self {
|
||||||
|
max_photos: config.max_photos,
|
||||||
|
time_range: config.time_range,
|
||||||
|
buckets: RwLock::new(HashMap::new()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Try to claim a slot for the given timestamp.
|
||||||
|
/// Returns true if the slot was claimed, false if the bucket is full.
|
||||||
|
pub fn try_claim(&self, timestamp: &str) -> bool {
|
||||||
|
let key = self.bucket_key(timestamp);
|
||||||
|
|
||||||
|
// Get or create the atomic counter for this bucket
|
||||||
|
let counter = {
|
||||||
|
// Try read lock first
|
||||||
|
let buckets = self.buckets.read().unwrap();
|
||||||
|
if let Some(counter) = buckets.get(&key) {
|
||||||
|
counter.clone()
|
||||||
|
} else {
|
||||||
|
drop(buckets);
|
||||||
|
let mut buckets = self.buckets.write().unwrap();
|
||||||
|
buckets
|
||||||
|
.entry(key)
|
||||||
|
.or_insert_with(|| Arc::new(AtomicU32::new(0)))
|
||||||
|
.clone()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Atomically try to claim: increment, check, undo if over limit
|
||||||
|
let prev = counter.fetch_add(1, Ordering::SeqCst);
|
||||||
|
if prev < self.max_photos {
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
counter.fetch_sub(1, Ordering::SeqCst);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compute the bucket key from a timestamp string.
|
||||||
|
fn bucket_key(&self, timestamp: &str) -> String {
|
||||||
|
// Parse date from ISO 8601 timestamp (e.g. "2024-01-15" or "2024-01-15T12:34:56Z")
|
||||||
|
let date_str = ×tamp[..timestamp.len().min(10)];
|
||||||
|
let date = NaiveDate::parse_from_str(date_str, "%Y-%m-%d").unwrap_or_default();
|
||||||
|
|
||||||
|
match self.time_range {
|
||||||
|
TimeRange::Day => date.format("%Y-%m-%d").to_string(),
|
||||||
|
TimeRange::Week => {
|
||||||
|
use chrono::Datelike;
|
||||||
|
format!("{}-W{:02}", date.iso_week().year(), date.iso_week().week())
|
||||||
|
}
|
||||||
|
TimeRange::Month => date.format("%Y-%m").to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Parameters for starting a processing job.
|
/// Parameters for starting a processing job.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct JobParams {
|
pub struct JobParams {
|
||||||
|
|
@ -202,6 +276,9 @@ async fn run_job_inner(
|
||||||
let pipeline = Arc::new(Pipeline::with_steps_from_config(&config));
|
let pipeline = Arc::new(Pipeline::with_steps_from_config(&config));
|
||||||
tracing::debug!("Pipeline steps: {:?}", pipeline.step_ids());
|
tracing::debug!("Pipeline steps: {:?}", pipeline.step_ids());
|
||||||
|
|
||||||
|
// Create photo limit tracker if enabled
|
||||||
|
let photo_limit_tracker = PhotoLimitTracker::new(&config.processing.photo_limit).map(Arc::new);
|
||||||
|
|
||||||
// Process images in parallel with concurrency limit
|
// Process images in parallel with concurrency limit
|
||||||
let completed = Arc::new(AtomicU32::new(0));
|
let completed = Arc::new(AtomicU32::new(0));
|
||||||
let semaphore = Arc::new(Semaphore::new(config.processing.max_workers));
|
let semaphore = Arc::new(Semaphore::new(config.processing.max_workers));
|
||||||
|
|
@ -239,6 +316,9 @@ async fn run_job_inner(
|
||||||
// Clone skip stats for this task
|
// Clone skip stats for this task
|
||||||
let skip_stats = skip_stats.clone();
|
let skip_stats = skip_stats.clone();
|
||||||
|
|
||||||
|
// Clone photo limit tracker for this task
|
||||||
|
let photo_limit_tracker = photo_limit_tracker.clone();
|
||||||
|
|
||||||
// Clone person info for this task
|
// Clone person info for this task
|
||||||
let person_id = task_person_id.clone();
|
let person_id = task_person_id.clone();
|
||||||
let person_name = task_person_name.clone();
|
let person_name = task_person_name.clone();
|
||||||
|
|
@ -261,6 +341,7 @@ async fn run_job_inner(
|
||||||
&task_cancel_token,
|
&task_cancel_token,
|
||||||
&skip_stats,
|
&skip_stats,
|
||||||
&pipeline,
|
&pipeline,
|
||||||
|
photo_limit_tracker.as_ref(),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,8 @@ use crate::immich_api::{Asset, FaceData, ImmichClient};
|
||||||
use crate::pipeline::{Pipeline, PipelineContext, PipelineResult};
|
use crate::pipeline::{Pipeline, PipelineContext, PipelineResult};
|
||||||
use crate::web::AtomicSkipStats;
|
use crate::web::AtomicSkipStats;
|
||||||
|
|
||||||
|
use super::PhotoLimitTracker;
|
||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use image::ImageFormat;
|
use image::ImageFormat;
|
||||||
use std::io::Cursor;
|
use std::io::Cursor;
|
||||||
|
|
@ -68,6 +70,7 @@ pub async fn process_single_asset(
|
||||||
cancel_token: &CancellationToken,
|
cancel_token: &CancellationToken,
|
||||||
skip_stats: &Arc<AtomicSkipStats>,
|
skip_stats: &Arc<AtomicSkipStats>,
|
||||||
pipeline: &Pipeline,
|
pipeline: &Pipeline,
|
||||||
|
photo_limit: Option<&Arc<PhotoLimitTracker>>,
|
||||||
) -> AssetProcessResult {
|
) -> AssetProcessResult {
|
||||||
let asset_id = &asset.id;
|
let asset_id = &asset.id;
|
||||||
|
|
||||||
|
|
@ -119,6 +122,17 @@ pub async fn process_single_asset(
|
||||||
timestamp,
|
timestamp,
|
||||||
..
|
..
|
||||||
} => {
|
} => {
|
||||||
|
// Check photo limit before saving
|
||||||
|
if let Some(tracker) = photo_limit {
|
||||||
|
if !tracker.try_claim(×tamp) {
|
||||||
|
skip_stats.increment("photo_limit");
|
||||||
|
return AssetProcessResult::Skipped {
|
||||||
|
asset_id,
|
||||||
|
reason: "Photo limit reached for time range".to_string(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Sanitize timestamp for filename
|
// Sanitize timestamp for filename
|
||||||
let safe_timestamp: String = timestamp
|
let safe_timestamp: String = timestamp
|
||||||
.chars()
|
.chars()
|
||||||
|
|
|
||||||
|
|
@ -252,13 +252,6 @@ impl ProcessingStep for HeadPoseStep {
|
||||||
// Check against thresholds
|
// Check against thresholds
|
||||||
let head_pose_config = &config.processing.head_pose;
|
let head_pose_config = &config.processing.head_pose;
|
||||||
|
|
||||||
tracing::debug!(
|
|
||||||
"Head pose detected: yaw={:.1}°, pitch={:.1}°, roll={:.1}°",
|
|
||||||
pose.yaw,
|
|
||||||
pose.pitch,
|
|
||||||
pose.roll
|
|
||||||
);
|
|
||||||
|
|
||||||
if pose.yaw.abs() > head_pose_config.max_yaw {
|
if pose.yaw.abs() > head_pose_config.max_yaw {
|
||||||
return StepOutcome::Skip {
|
return StepOutcome::Skip {
|
||||||
ctx,
|
ctx,
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
use crate::config::{
|
use crate::config::{
|
||||||
AlignmentConfig, BlurConfig, BrightnessConfig, EyeFilterConfig, FaceResolutionConfig,
|
AlignmentConfig, BlurConfig, BrightnessConfig, EyeFilterConfig, FaceResolutionConfig,
|
||||||
HeadPoseConfig, OutputConfig, ProcessingConfig, TimestampConfig, VideoConfig,
|
HeadPoseConfig, OutputConfig, PhotoLimitConfig, ProcessingConfig, TimestampConfig, VideoConfig,
|
||||||
};
|
};
|
||||||
use crate::web::state::AppState;
|
use crate::web::state::AppState;
|
||||||
use axum::{extract::State, http::StatusCode, response::Json};
|
use axum::{extract::State, http::StatusCode, response::Json};
|
||||||
|
|
@ -46,6 +46,7 @@ pub struct ProcessingConfigUpdate {
|
||||||
pub output: Option<OutputConfig>,
|
pub output: Option<OutputConfig>,
|
||||||
pub alignment: Option<AlignmentConfig>,
|
pub alignment: Option<AlignmentConfig>,
|
||||||
pub timestamp: Option<TimestampConfig>,
|
pub timestamp: Option<TimestampConfig>,
|
||||||
|
pub photo_limit: Option<PhotoLimitConfig>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Video configuration update fields.
|
/// Video configuration update fields.
|
||||||
|
|
@ -177,6 +178,21 @@ fn validate_processing_config(proc: &ProcessingConfigUpdate) -> Result<(), Valid
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let Some(ref pl) = proc.photo_limit {
|
||||||
|
if pl.enabled && pl.max_photos == 0 {
|
||||||
|
return Err(ValidationError::new(
|
||||||
|
"processing.photo_limit.max_photos",
|
||||||
|
"must be greater than 0 when enabled",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if pl.enabled && pl.max_photos > 100 {
|
||||||
|
return Err(ValidationError::new(
|
||||||
|
"processing.photo_limit.max_photos",
|
||||||
|
format!("must be at most 100, got {}", pl.max_photos),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -270,6 +286,9 @@ pub async fn update_config(
|
||||||
if let Some(v) = proc.timestamp {
|
if let Some(v) = proc.timestamp {
|
||||||
config.processing.timestamp = v;
|
config.processing.timestamp = v;
|
||||||
}
|
}
|
||||||
|
if let Some(v) = proc.photo_limit {
|
||||||
|
config.processing.photo_limit = v;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(vid) = update.video {
|
if let Some(vid) = update.video {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue