Add date proximity limitting
This commit is contained in:
parent
fb4fc59e84
commit
0ae09da074
7 changed files with 228 additions and 9 deletions
|
|
@ -78,6 +78,11 @@
|
|||
month: config.processing.timestamp.month,
|
||||
day: config.processing.timestamp.day,
|
||||
},
|
||||
photo_limit: {
|
||||
enabled: config.processing.photo_limit.enabled,
|
||||
max_photos: Number(config.processing.photo_limit.max_photos),
|
||||
time_range: config.processing.photo_limit.time_range,
|
||||
},
|
||||
},
|
||||
video: {
|
||||
enabled: config.video.enabled,
|
||||
|
|
@ -408,6 +413,49 @@
|
|||
<input id="keep-intermediates" type="checkbox" bind:checked={config.processing.output.keep_intermediates} />
|
||||
</div>
|
||||
|
||||
<!-- Photo Limit Section -->
|
||||
<div class="setting-section">
|
||||
<div class="section-header">
|
||||
<span class="section-title">Photo Limit</span>
|
||||
<input
|
||||
type="checkbox"
|
||||
bind:checked={config.processing.photo_limit.enabled}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{#if config.processing.photo_limit.enabled}
|
||||
<div class="setting-row sub-setting">
|
||||
<label for="max-photos">
|
||||
<span class="setting-label">Max Photos</span>
|
||||
<span class="setting-hint">Maximum photos per time range</span>
|
||||
</label>
|
||||
<div class="setting-control">
|
||||
<input
|
||||
id="max-photos"
|
||||
type="range"
|
||||
bind:value={config.processing.photo_limit.max_photos}
|
||||
min="1"
|
||||
max="10"
|
||||
step="1"
|
||||
/>
|
||||
<span class="value">{config.processing.photo_limit.max_photos}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="setting-row sub-setting">
|
||||
<label for="time-range">
|
||||
<span class="setting-label">Time Range</span>
|
||||
<span class="setting-hint">Group photos by this period</span>
|
||||
</label>
|
||||
<select id="time-range" bind:value={config.processing.photo_limit.time_range}>
|
||||
<option value="day">Day</option>
|
||||
<option value="week">Week</option>
|
||||
<option value="month">Month</option>
|
||||
</select>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<!-- Timestamp Section -->
|
||||
<div class="setting-section">
|
||||
<div class="section-header">
|
||||
|
|
|
|||
|
|
@ -166,6 +166,11 @@ export const DEFAULT_CONFIG = {
|
|||
month: false,
|
||||
day: false,
|
||||
},
|
||||
photo_limit: {
|
||||
enabled: false,
|
||||
max_photos: 1,
|
||||
time_range: 'day',
|
||||
},
|
||||
},
|
||||
video: {
|
||||
enabled: true,
|
||||
|
|
|
|||
|
|
@ -445,6 +445,59 @@ impl TimestampConfig {
|
|||
}
|
||||
}
|
||||
|
||||
/// Time range granularity for photo limiting.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TimeRange {
|
||||
#[default]
|
||||
Day,
|
||||
Week,
|
||||
Month,
|
||||
}
|
||||
|
||||
/// Photo limit configuration.
|
||||
///
|
||||
/// Limits the number of successfully processed photos per time range
|
||||
/// (day/week/month) to avoid over-representation of busy periods.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PhotoLimitConfig {
|
||||
/// Whether photo limiting is enabled.
|
||||
pub enabled: bool,
|
||||
|
||||
/// Maximum number of photos per time range.
|
||||
pub max_photos: u32,
|
||||
|
||||
/// Time range granularity.
|
||||
pub time_range: TimeRange,
|
||||
}
|
||||
|
||||
impl Default for PhotoLimitConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
max_photos: 1,
|
||||
time_range: TimeRange::Day,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PhotoLimitConfig {
|
||||
/// Validate the configuration values.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.enabled && self.max_photos == 0 {
|
||||
return Err(Error::Config(
|
||||
"Photo limit max_photos must be greater than 0".to_string(),
|
||||
));
|
||||
}
|
||||
if self.enabled && self.max_photos > 100 {
|
||||
return Err(Error::Config(
|
||||
"Photo limit max_photos must be at most 100".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main Processing Configuration
|
||||
// ============================================================================
|
||||
|
|
@ -490,6 +543,10 @@ pub struct ProcessingConfig {
|
|||
/// Timestamp overlay settings.
|
||||
#[serde(default)]
|
||||
pub timestamp: TimestampConfig,
|
||||
|
||||
/// Photo limit settings.
|
||||
#[serde(default)]
|
||||
pub photo_limit: PhotoLimitConfig,
|
||||
}
|
||||
|
||||
impl Default for ProcessingConfig {
|
||||
|
|
@ -504,6 +561,7 @@ impl Default for ProcessingConfig {
|
|||
output: OutputConfig::default(),
|
||||
alignment: AlignmentConfig::default(),
|
||||
timestamp: TimestampConfig::default(),
|
||||
photo_limit: PhotoLimitConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -519,6 +577,7 @@ impl ProcessingConfig {
|
|||
self.output.validate()?;
|
||||
self.alignment.validate()?;
|
||||
self.timestamp.validate()?;
|
||||
self.photo_limit.validate()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
mod processing;
|
||||
|
||||
use crate::config::{PhotoLimitConfig, TimeRange};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::immich_api::{Asset, FaceData, ImmichClient};
|
||||
use crate::pipeline::Pipeline;
|
||||
|
|
@ -17,12 +18,85 @@ use crate::web::{AppState, AtomicSkipStats, JobStatus, Progress, SkipStats};
|
|||
|
||||
use processing::{process_single_asset, AssetProcessResult, DebugDirs, OutputDirs};
|
||||
|
||||
use chrono::NaiveDate;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
/// Tracks per-time-bucket photo counts for limiting.
|
||||
///
|
||||
/// Thread-safe: uses atomic counters with fetch_add + undo pattern
|
||||
/// so concurrent workers can claim slots without races.
|
||||
pub struct PhotoLimitTracker {
|
||||
max_photos: u32,
|
||||
time_range: TimeRange,
|
||||
buckets: RwLock<HashMap<String, Arc<AtomicU32>>>,
|
||||
}
|
||||
|
||||
impl PhotoLimitTracker {
|
||||
/// Create a tracker if photo limiting is enabled, otherwise return None.
|
||||
pub fn new(config: &PhotoLimitConfig) -> Option<Self> {
|
||||
if !config.enabled {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
max_photos: config.max_photos,
|
||||
time_range: config.time_range,
|
||||
buckets: RwLock::new(HashMap::new()),
|
||||
})
|
||||
}
|
||||
|
||||
/// Try to claim a slot for the given timestamp.
|
||||
/// Returns true if the slot was claimed, false if the bucket is full.
|
||||
pub fn try_claim(&self, timestamp: &str) -> bool {
|
||||
let key = self.bucket_key(timestamp);
|
||||
|
||||
// Get or create the atomic counter for this bucket
|
||||
let counter = {
|
||||
// Try read lock first
|
||||
let buckets = self.buckets.read().unwrap();
|
||||
if let Some(counter) = buckets.get(&key) {
|
||||
counter.clone()
|
||||
} else {
|
||||
drop(buckets);
|
||||
let mut buckets = self.buckets.write().unwrap();
|
||||
buckets
|
||||
.entry(key)
|
||||
.or_insert_with(|| Arc::new(AtomicU32::new(0)))
|
||||
.clone()
|
||||
}
|
||||
};
|
||||
|
||||
// Atomically try to claim: increment, check, undo if over limit
|
||||
let prev = counter.fetch_add(1, Ordering::SeqCst);
|
||||
if prev < self.max_photos {
|
||||
true
|
||||
} else {
|
||||
counter.fetch_sub(1, Ordering::SeqCst);
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the bucket key from a timestamp string.
|
||||
fn bucket_key(&self, timestamp: &str) -> String {
|
||||
// Parse date from ISO 8601 timestamp (e.g. "2024-01-15" or "2024-01-15T12:34:56Z")
|
||||
let date_str = ×tamp[..timestamp.len().min(10)];
|
||||
let date = NaiveDate::parse_from_str(date_str, "%Y-%m-%d").unwrap_or_default();
|
||||
|
||||
match self.time_range {
|
||||
TimeRange::Day => date.format("%Y-%m-%d").to_string(),
|
||||
TimeRange::Week => {
|
||||
use chrono::Datelike;
|
||||
format!("{}-W{:02}", date.iso_week().year(), date.iso_week().week())
|
||||
}
|
||||
TimeRange::Month => date.format("%Y-%m").to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters for starting a processing job.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct JobParams {
|
||||
|
|
@ -202,6 +276,9 @@ async fn run_job_inner(
|
|||
let pipeline = Arc::new(Pipeline::with_steps_from_config(&config));
|
||||
tracing::debug!("Pipeline steps: {:?}", pipeline.step_ids());
|
||||
|
||||
// Create photo limit tracker if enabled
|
||||
let photo_limit_tracker = PhotoLimitTracker::new(&config.processing.photo_limit).map(Arc::new);
|
||||
|
||||
// Process images in parallel with concurrency limit
|
||||
let completed = Arc::new(AtomicU32::new(0));
|
||||
let semaphore = Arc::new(Semaphore::new(config.processing.max_workers));
|
||||
|
|
@ -239,6 +316,9 @@ async fn run_job_inner(
|
|||
// Clone skip stats for this task
|
||||
let skip_stats = skip_stats.clone();
|
||||
|
||||
// Clone photo limit tracker for this task
|
||||
let photo_limit_tracker = photo_limit_tracker.clone();
|
||||
|
||||
// Clone person info for this task
|
||||
let person_id = task_person_id.clone();
|
||||
let person_name = task_person_name.clone();
|
||||
|
|
@ -261,6 +341,7 @@ async fn run_job_inner(
|
|||
&task_cancel_token,
|
||||
&skip_stats,
|
||||
&pipeline,
|
||||
photo_limit_tracker.as_ref(),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ use crate::immich_api::{Asset, FaceData, ImmichClient};
|
|||
use crate::pipeline::{Pipeline, PipelineContext, PipelineResult};
|
||||
use crate::web::AtomicSkipStats;
|
||||
|
||||
use super::PhotoLimitTracker;
|
||||
|
||||
use bytes::Bytes;
|
||||
use image::ImageFormat;
|
||||
use std::io::Cursor;
|
||||
|
|
@ -68,6 +70,7 @@ pub async fn process_single_asset(
|
|||
cancel_token: &CancellationToken,
|
||||
skip_stats: &Arc<AtomicSkipStats>,
|
||||
pipeline: &Pipeline,
|
||||
photo_limit: Option<&Arc<PhotoLimitTracker>>,
|
||||
) -> AssetProcessResult {
|
||||
let asset_id = &asset.id;
|
||||
|
||||
|
|
@ -119,6 +122,17 @@ pub async fn process_single_asset(
|
|||
timestamp,
|
||||
..
|
||||
} => {
|
||||
// Check photo limit before saving
|
||||
if let Some(tracker) = photo_limit {
|
||||
if !tracker.try_claim(×tamp) {
|
||||
skip_stats.increment("photo_limit");
|
||||
return AssetProcessResult::Skipped {
|
||||
asset_id,
|
||||
reason: "Photo limit reached for time range".to_string(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Sanitize timestamp for filename
|
||||
let safe_timestamp: String = timestamp
|
||||
.chars()
|
||||
|
|
|
|||
|
|
@ -252,13 +252,6 @@ impl ProcessingStep for HeadPoseStep {
|
|||
// Check against thresholds
|
||||
let head_pose_config = &config.processing.head_pose;
|
||||
|
||||
tracing::debug!(
|
||||
"Head pose detected: yaw={:.1}°, pitch={:.1}°, roll={:.1}°",
|
||||
pose.yaw,
|
||||
pose.pitch,
|
||||
pose.roll
|
||||
);
|
||||
|
||||
if pose.yaw.abs() > head_pose_config.max_yaw {
|
||||
return StepOutcome::Skip {
|
||||
ctx,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
use crate::config::{
|
||||
AlignmentConfig, BlurConfig, BrightnessConfig, EyeFilterConfig, FaceResolutionConfig,
|
||||
HeadPoseConfig, OutputConfig, ProcessingConfig, TimestampConfig, VideoConfig,
|
||||
HeadPoseConfig, OutputConfig, PhotoLimitConfig, ProcessingConfig, TimestampConfig, VideoConfig,
|
||||
};
|
||||
use crate::web::state::AppState;
|
||||
use axum::{extract::State, http::StatusCode, response::Json};
|
||||
|
|
@ -46,6 +46,7 @@ pub struct ProcessingConfigUpdate {
|
|||
pub output: Option<OutputConfig>,
|
||||
pub alignment: Option<AlignmentConfig>,
|
||||
pub timestamp: Option<TimestampConfig>,
|
||||
pub photo_limit: Option<PhotoLimitConfig>,
|
||||
}
|
||||
|
||||
/// Video configuration update fields.
|
||||
|
|
@ -177,6 +178,21 @@ fn validate_processing_config(proc: &ProcessingConfigUpdate) -> Result<(), Valid
|
|||
}
|
||||
}
|
||||
|
||||
if let Some(ref pl) = proc.photo_limit {
|
||||
if pl.enabled && pl.max_photos == 0 {
|
||||
return Err(ValidationError::new(
|
||||
"processing.photo_limit.max_photos",
|
||||
"must be greater than 0 when enabled",
|
||||
));
|
||||
}
|
||||
if pl.enabled && pl.max_photos > 100 {
|
||||
return Err(ValidationError::new(
|
||||
"processing.photo_limit.max_photos",
|
||||
format!("must be at most 100, got {}", pl.max_photos),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -270,6 +286,9 @@ pub async fn update_config(
|
|||
if let Some(v) = proc.timestamp {
|
||||
config.processing.timestamp = v;
|
||||
}
|
||||
if let Some(v) = proc.photo_limit {
|
||||
config.processing.photo_limit = v;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(vid) = update.video {
|
||||
|
|
|
|||
Loading…
Reference in a new issue