Expired Download Cleaner: retention-based cleanup of watchlist/playlist downloads (Boulder)
A Library Maintenance job that cleans up downloads tracked by Download Origins
once they pass a per-origin retention window — findings by default, opt-in
auto-delete.
A download is only ever proposed for deletion when ALL hold: older than its
origin's retention, NOT still in an actively-mirrored playlist / watched
artist, and played fewer than the keep-threshold (default 2 → "played more
than once is kept"). Only touches downloads recorded from the Download Origins
feature forward — never pre-existing or manual library.
- core/library/expired_cleanup.py: pure decision core (retention_cutoff,
is_expired, select_expired) — no DB/clock, fully tested. play_count is the
reliable listen signal (last_played is often unpopulated, so recency isn't
used).
- ExpiredDownloadCleanerJob: gathers facts (play_count via a new
get_origin_cleanup_candidates join; active-mirror via get_mirrored_playlists;
watch via get_watchlist_artists) and either creates 'expired_download'
findings or, with auto_delete on, deletes in-scan. Default OFF, both
retentions default 'off'. Settings auto-render in the Library Maintenance
panel (same as Cover Art / Lyrics / Re-tag).
- delete_origin_download(): shared delete (resolve path → remove file → drop
track row → drop history row); a file that won't delete keeps its row +
reports. Used by auto mode AND the _fix_expired_download apply handler.
- Frontend: type/action ('Delete')/result labels + finding detail render.
Tests: 9 on the pure brain (windows, off, per-origin, protected, play-count
threshold, bad age) + 7 on the job (no-op when off, findings, mirror/watch
protection, auto-delete, delete helper missing/real file). 185 repair/origin
tests pass.
This commit is contained in:
parent
6c3e285a49
commit
696119d5ac
8 changed files with 579 additions and 2 deletions
98
core/library/expired_cleanup.py
Normal file
98
core/library/expired_cleanup.py
Normal file
|
|
@ -0,0 +1,98 @@
|
||||||
|
"""Pure expiry decision for the Expired Download Cleaner job.
|
||||||
|
|
||||||
|
Decides which origin-tracked downloads (watchlist / playlist, recorded by the
|
||||||
|
Download Origins provenance) are past their retention window and safe to
|
||||||
|
propose for deletion. No DB, no clock, no I/O — the job annotates each entry
|
||||||
|
with the facts (play_count, whether it's still in an active mirror) and this
|
||||||
|
module decides. Fully unit-testable.
|
||||||
|
|
||||||
|
A download is proposed for deletion ONLY when ALL hold:
|
||||||
|
- its origin's retention is set (not 'off') and it's older than that window,
|
||||||
|
- it's NOT protected (still in an actively-mirrored playlist / watched artist),
|
||||||
|
- it has been played FEWER than ``min_plays`` times (default 2 → "played more
|
||||||
|
than once is kept"; play_count is the reliable signal, last_played is not).
|
||||||
|
|
||||||
|
Anything failing a check is kept. Deliberately conservative — this deletes the
|
||||||
|
user's files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from typing import Any, Dict, Iterable, List, Optional
|
||||||
|
|
||||||
|
# Retention option → days. 'off' (or anything unmapped) disables that origin.
|
||||||
|
RETENTION_DAYS = {
|
||||||
|
"1w": 7, "2w": 14, "3w": 21, "4w": 28,
|
||||||
|
"2mo": 60, "3mo": 90, "6mo": 180,
|
||||||
|
}
|
||||||
|
RETENTION_OPTIONS = ["off", "1w", "2w", "3w", "4w", "2mo", "3mo", "6mo"]
|
||||||
|
|
||||||
|
|
||||||
|
def retention_cutoff(retention: Optional[str], now: datetime) -> Optional[datetime]:
|
||||||
|
"""Datetime before which an entry of this retention is expired, or None
|
||||||
|
when the retention is off/unknown (origin never auto-cleaned)."""
|
||||||
|
days = RETENTION_DAYS.get((retention or "").strip().lower())
|
||||||
|
if not days:
|
||||||
|
return None
|
||||||
|
return now - timedelta(days=days)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_ts(value: Any) -> Optional[datetime]:
|
||||||
|
"""Parse a SQLite CURRENT_TIMESTAMP (UTC, no zone) or ISO string."""
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
text = str(value).strip().replace(" ", "T")
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(text)
|
||||||
|
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def is_expired(
|
||||||
|
entry: Dict[str, Any],
|
||||||
|
*,
|
||||||
|
watchlist_retention: Optional[str],
|
||||||
|
playlist_retention: Optional[str],
|
||||||
|
min_plays: int,
|
||||||
|
now: datetime,
|
||||||
|
) -> bool:
|
||||||
|
"""True if this origin entry should be proposed for deletion.
|
||||||
|
|
||||||
|
``entry`` needs: ``origin`` ('watchlist'|'playlist'), ``created_at``,
|
||||||
|
``play_count`` (int, may be None), ``protected`` (bool — still in an active
|
||||||
|
mirror/watch)."""
|
||||||
|
if entry.get("protected"):
|
||||||
|
return False
|
||||||
|
if (entry.get("play_count") or 0) >= max(1, int(min_plays or 1)):
|
||||||
|
return False # listened to enough to keep
|
||||||
|
origin = (entry.get("origin") or "").strip().lower()
|
||||||
|
retention = watchlist_retention if origin == "watchlist" else playlist_retention
|
||||||
|
cutoff = retention_cutoff(retention, now)
|
||||||
|
if cutoff is None:
|
||||||
|
return False # this origin's auto-clean is off
|
||||||
|
created = _parse_ts(entry.get("created_at"))
|
||||||
|
if created is None:
|
||||||
|
return False # unknown age → never delete
|
||||||
|
return created < cutoff
|
||||||
|
|
||||||
|
|
||||||
|
def select_expired(
|
||||||
|
entries: Iterable[Dict[str, Any]],
|
||||||
|
*,
|
||||||
|
watchlist_retention: Optional[str],
|
||||||
|
playlist_retention: Optional[str],
|
||||||
|
min_plays: int = 2,
|
||||||
|
now: Optional[datetime] = None,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Return the subset of ``entries`` that are expired + safe to delete."""
|
||||||
|
now = now or datetime.now(timezone.utc)
|
||||||
|
return [
|
||||||
|
e for e in (entries or [])
|
||||||
|
if is_expired(e, watchlist_retention=watchlist_retention,
|
||||||
|
playlist_retention=playlist_retention,
|
||||||
|
min_plays=min_plays, now=now)
|
||||||
|
]
|
||||||
|
|
@ -34,6 +34,7 @@ _JOB_MODULES = [
|
||||||
'core.repair_jobs.acoustid_scanner',
|
'core.repair_jobs.acoustid_scanner',
|
||||||
'core.repair_jobs.missing_cover_art',
|
'core.repair_jobs.missing_cover_art',
|
||||||
'core.repair_jobs.missing_lyrics',
|
'core.repair_jobs.missing_lyrics',
|
||||||
|
'core.repair_jobs.expired_download_cleaner',
|
||||||
'core.repair_jobs.metadata_gap_filler',
|
'core.repair_jobs.metadata_gap_filler',
|
||||||
'core.repair_jobs.album_completeness',
|
'core.repair_jobs.album_completeness',
|
||||||
'core.repair_jobs.fake_lossless_detector',
|
'core.repair_jobs.fake_lossless_detector',
|
||||||
|
|
|
||||||
201
core/repair_jobs/expired_download_cleaner.py
Normal file
201
core/repair_jobs/expired_download_cleaner.py
Normal file
|
|
@ -0,0 +1,201 @@
|
||||||
|
"""Expired Download Cleaner (Boulder) — retention-based cleanup of
|
||||||
|
origin-tracked downloads.
|
||||||
|
|
||||||
|
Watchlist- and playlist-sourced downloads (recorded by the Download Origins
|
||||||
|
provenance) get a per-origin retention window. Past it, a download is proposed
|
||||||
|
for deletion UNLESS it's still in an actively-mirrored playlist / watched
|
||||||
|
artist, or you've played it more than once. By default it creates findings to
|
||||||
|
review; flip ``auto_delete`` to true for hands-off cleanup.
|
||||||
|
|
||||||
|
The expiry decision is the pure core in core.library.expired_cleanup; this job
|
||||||
|
gathers the facts (play_count via DB, active-mirror/watch protection) and
|
||||||
|
deletes via the shared helper the Download Origins delete also conceptually
|
||||||
|
uses (resolve path → remove file → drop track row → drop history row).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from core.library.expired_cleanup import RETENTION_OPTIONS, select_expired
|
||||||
|
from core.library.path_resolver import resolve_library_file_path
|
||||||
|
from core.repair_jobs import register_job
|
||||||
|
from core.repair_jobs.base import JobContext, JobResult, RepairJob
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("repair_jobs.expired_download_cleaner")
|
||||||
|
|
||||||
|
|
||||||
|
def delete_origin_download(db, entry, config_manager) -> dict:
|
||||||
|
"""Delete one origin-tracked download: the file on disk (resolved through
|
||||||
|
the shared resolver), its library track row, and the history entry. A file
|
||||||
|
that refuses deletion keeps its history row and reports the error. Returns
|
||||||
|
{removed, file_deleted, error}."""
|
||||||
|
raw_path = entry.get('file_path') or ''
|
||||||
|
file_deleted = False
|
||||||
|
error = None
|
||||||
|
if raw_path:
|
||||||
|
resolved = resolve_library_file_path(raw_path, config_manager=config_manager)
|
||||||
|
if resolved and os.path.isfile(resolved):
|
||||||
|
try:
|
||||||
|
os.remove(resolved)
|
||||||
|
file_deleted = True
|
||||||
|
except OSError as e:
|
||||||
|
error = str(e)
|
||||||
|
# File gone or deleted → clean up the library track row either way.
|
||||||
|
if error is None:
|
||||||
|
try:
|
||||||
|
db.delete_track_by_file_path(raw_path)
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("expired cleanup: track row delete failed: %s", e)
|
||||||
|
removed = 0
|
||||||
|
if error is None:
|
||||||
|
removed = db.delete_library_history_rows([entry['id']])
|
||||||
|
return {'removed': removed, 'file_deleted': file_deleted, 'error': error}
|
||||||
|
|
||||||
|
|
||||||
|
@register_job
|
||||||
|
class ExpiredDownloadCleanerJob(RepairJob):
|
||||||
|
job_id = 'expired_download_cleaner'
|
||||||
|
display_name = 'Expired Download Cleaner'
|
||||||
|
description = 'Deletes watchlist/playlist downloads past a retention window (keeps active + played ones)'
|
||||||
|
help_text = (
|
||||||
|
'Cleans up downloads that came in via the watchlist or playlist sync '
|
||||||
|
'(tracked by Download Origins) once they pass a retention window you set '
|
||||||
|
'per origin.\n\n'
|
||||||
|
'A download is only ever proposed for deletion when ALL are true: it is '
|
||||||
|
'older than its origin\'s retention, it is NOT still in a playlist you '
|
||||||
|
'actively mirror (or an artist you still watch), and you have played it '
|
||||||
|
'fewer than the keep-threshold (default: played more than once is kept). '
|
||||||
|
'It only touches downloads recorded from the Download Origins feature '
|
||||||
|
'forward — never your pre-existing or manually-added library.\n\n'
|
||||||
|
'By default it creates findings for you to review and delete. Set '
|
||||||
|
'Auto-delete to true for hands-off cleanup.\n\n'
|
||||||
|
'Settings:\n'
|
||||||
|
'- Watchlist retention / Playlist retention: off, or a window\n'
|
||||||
|
'- Keep if played at least: play count that protects a track (default 2)\n'
|
||||||
|
'- Auto-delete: delete automatically instead of creating findings'
|
||||||
|
)
|
||||||
|
icon = 'repair-icon-cleanup'
|
||||||
|
default_enabled = False
|
||||||
|
default_interval_hours = 24
|
||||||
|
default_settings = {
|
||||||
|
'watchlist_retention': 'off',
|
||||||
|
'playlist_retention': 'off',
|
||||||
|
'keep_if_played_at_least': 2,
|
||||||
|
'auto_delete': False,
|
||||||
|
}
|
||||||
|
setting_options = {
|
||||||
|
'watchlist_retention': RETENTION_OPTIONS,
|
||||||
|
'playlist_retention': RETENTION_OPTIONS,
|
||||||
|
'auto_delete': [False, True],
|
||||||
|
}
|
||||||
|
auto_fix = False
|
||||||
|
|
||||||
|
def _get_settings(self, context: JobContext) -> dict:
|
||||||
|
merged = dict(self.default_settings)
|
||||||
|
if context.config_manager:
|
||||||
|
cfg = context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {}) or {}
|
||||||
|
merged.update(cfg)
|
||||||
|
return merged
|
||||||
|
|
||||||
|
def scan(self, context: JobContext) -> JobResult:
|
||||||
|
result = JobResult()
|
||||||
|
settings = self._get_settings(context)
|
||||||
|
wl = (settings.get('watchlist_retention') or 'off')
|
||||||
|
pl = (settings.get('playlist_retention') or 'off')
|
||||||
|
if wl == 'off' and pl == 'off':
|
||||||
|
return result # nothing configured — no-op
|
||||||
|
try:
|
||||||
|
min_plays = int(settings.get('keep_if_played_at_least', 2))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
min_plays = 2
|
||||||
|
auto_delete = bool(settings.get('auto_delete', False))
|
||||||
|
|
||||||
|
candidates = context.db.get_origin_cleanup_candidates()
|
||||||
|
if not candidates:
|
||||||
|
return result
|
||||||
|
|
||||||
|
# Build the "protected" set: still-mirrored playlists + still-watched
|
||||||
|
# artists (by name — what origin_context stores). Case-folded.
|
||||||
|
mirrored_names, watched_names = set(), set()
|
||||||
|
try:
|
||||||
|
for p in (context.db.get_mirrored_playlists() or []):
|
||||||
|
n = (p.get('name') if isinstance(p, dict) else None) or ''
|
||||||
|
if n:
|
||||||
|
mirrored_names.add(n.strip().casefold())
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("expired cleanup: mirrored-playlist lookup failed: %s", e)
|
||||||
|
try:
|
||||||
|
for a in (context.db.get_watchlist_artists() or []):
|
||||||
|
n = getattr(a, 'artist_name', None) or ''
|
||||||
|
if n:
|
||||||
|
watched_names.add(n.strip().casefold())
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("expired cleanup: watchlist lookup failed: %s", e)
|
||||||
|
|
||||||
|
for c in candidates:
|
||||||
|
ctx = (c.get('origin_context') or '').strip().casefold()
|
||||||
|
origin = (c.get('origin') or '').strip().lower()
|
||||||
|
c['protected'] = bool(
|
||||||
|
(origin == 'playlist' and ctx and ctx in mirrored_names) or
|
||||||
|
(origin == 'watchlist' and ctx and ctx in watched_names))
|
||||||
|
|
||||||
|
expired = select_expired(candidates, watchlist_retention=wl,
|
||||||
|
playlist_retention=pl, min_plays=min_plays)
|
||||||
|
result.scanned = len(candidates)
|
||||||
|
if context.update_progress:
|
||||||
|
context.update_progress(0, len(expired))
|
||||||
|
|
||||||
|
for i, entry in enumerate(expired):
|
||||||
|
if context.check_stop():
|
||||||
|
return result
|
||||||
|
if auto_delete:
|
||||||
|
try:
|
||||||
|
res = delete_origin_download(context.db, entry, context.config_manager)
|
||||||
|
if res.get('removed') or res.get('file_deleted'):
|
||||||
|
result.auto_fixed += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("expired auto-delete failed for %s: %s", entry.get('title'), e)
|
||||||
|
result.errors += 1
|
||||||
|
elif context.create_finding:
|
||||||
|
try:
|
||||||
|
inserted = context.create_finding(
|
||||||
|
job_id=self.job_id,
|
||||||
|
finding_type='expired_download',
|
||||||
|
severity='info',
|
||||||
|
entity_type='track',
|
||||||
|
entity_id=str(entry.get('id')),
|
||||||
|
file_path=entry.get('file_path'),
|
||||||
|
title=f'Expired: {entry.get("title") or "Unknown"}',
|
||||||
|
description=(f'"{entry.get("title")}" by {entry.get("artist_name") or "Unknown"} '
|
||||||
|
f'— via {entry.get("origin")} ({entry.get("origin_context") or "?"}), '
|
||||||
|
f'past retention, not active, not replayed.'),
|
||||||
|
details={
|
||||||
|
'history_id': entry.get('id'),
|
||||||
|
'file_path': entry.get('file_path'),
|
||||||
|
'title': entry.get('title'),
|
||||||
|
'artist': entry.get('artist_name'),
|
||||||
|
'origin': entry.get('origin'),
|
||||||
|
'origin_context': entry.get('origin_context'),
|
||||||
|
})
|
||||||
|
if inserted:
|
||||||
|
result.findings_created += 1
|
||||||
|
else:
|
||||||
|
result.findings_skipped_dedup += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("expired finding create failed: %s", e)
|
||||||
|
result.errors += 1
|
||||||
|
if context.update_progress and (i + 1) % 5 == 0:
|
||||||
|
context.update_progress(i + 1, len(expired))
|
||||||
|
|
||||||
|
logger.info("[Expired Cleaner] %d candidates, %d expired (%s)",
|
||||||
|
len(candidates), len(expired),
|
||||||
|
"auto-deleted" if auto_delete else "findings created")
|
||||||
|
return result
|
||||||
|
|
||||||
|
def estimate_scope(self, context: JobContext) -> int:
|
||||||
|
try:
|
||||||
|
return len(context.db.get_origin_cleanup_candidates())
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
@ -964,6 +964,7 @@ class RepairWorker:
|
||||||
'track_number_mismatch': self._fix_track_number,
|
'track_number_mismatch': self._fix_track_number,
|
||||||
'missing_cover_art': self._fix_missing_cover_art,
|
'missing_cover_art': self._fix_missing_cover_art,
|
||||||
'missing_lyrics': self._fix_missing_lyrics,
|
'missing_lyrics': self._fix_missing_lyrics,
|
||||||
|
'expired_download': self._fix_expired_download,
|
||||||
'metadata_gap': self._fix_metadata_gap,
|
'metadata_gap': self._fix_metadata_gap,
|
||||||
'duplicate_tracks': self._fix_duplicates,
|
'duplicate_tracks': self._fix_duplicates,
|
||||||
'single_album_redundant': self._fix_single_album_redundant,
|
'single_album_redundant': self._fix_single_album_redundant,
|
||||||
|
|
@ -1445,6 +1446,21 @@ class RepairWorker:
|
||||||
return {'success': False, 'error': 'Could not fetch lyrics (no longer available?)'}
|
return {'success': False, 'error': 'Could not fetch lyrics (no longer available?)'}
|
||||||
return {'success': True, 'action': 'applied_lyrics', 'message': 'Wrote lyrics (.lrc) + embedded'}
|
return {'success': True, 'action': 'applied_lyrics', 'message': 'Wrote lyrics (.lrc) + embedded'}
|
||||||
|
|
||||||
|
def _fix_expired_download(self, entity_type, entity_id, file_path, details):
|
||||||
|
"""Apply an expired-download finding: delete the file + library row +
|
||||||
|
history entry, via the same helper the cleaner's auto mode uses."""
|
||||||
|
from core.repair_jobs.expired_download_cleaner import delete_origin_download
|
||||||
|
entry = {'id': details.get('history_id') or entity_id,
|
||||||
|
'file_path': details.get('file_path') or file_path}
|
||||||
|
if not entry['id']:
|
||||||
|
return {'success': False, 'error': 'No history id in finding'}
|
||||||
|
res = delete_origin_download(self.db, entry, self._config_manager)
|
||||||
|
if res.get('error'):
|
||||||
|
return {'success': False, 'action': 'deleted_expired',
|
||||||
|
'error': f"Could not delete file: {res['error']}"}
|
||||||
|
verb = 'deleted file + entry' if res.get('file_deleted') else 'removed entry (file already gone)'
|
||||||
|
return {'success': True, 'action': 'deleted_expired', 'message': f'Expired download — {verb}'}
|
||||||
|
|
||||||
def _fix_library_retag(self, entity_type, entity_id, file_path, details):
|
def _fix_library_retag(self, entity_type, entity_id, file_path, details):
|
||||||
"""Apply a library re-tag finding: write each track's planned tags in
|
"""Apply a library re-tag finding: write each track's planned tags in
|
||||||
place (core.tag_writer.write_tags_to_file) + optionally embed/refresh
|
place (core.tag_writer.write_tags_to_file) + optionally embed/refresh
|
||||||
|
|
@ -3179,7 +3195,7 @@ class RepairWorker:
|
||||||
|
|
||||||
# Build query for pending fixable findings
|
# Build query for pending fixable findings
|
||||||
fixable_types = ('dead_file', 'orphan_file', 'track_number_mismatch',
|
fixable_types = ('dead_file', 'orphan_file', 'track_number_mismatch',
|
||||||
'missing_cover_art', 'missing_lyrics', 'metadata_gap', 'duplicate_tracks',
|
'missing_cover_art', 'missing_lyrics', 'expired_download', 'metadata_gap', 'duplicate_tracks',
|
||||||
'single_album_redundant', 'mbid_mismatch',
|
'single_album_redundant', 'mbid_mismatch',
|
||||||
'album_mbid_mismatch',
|
'album_mbid_mismatch',
|
||||||
'album_tag_inconsistency',
|
'album_tag_inconsistency',
|
||||||
|
|
|
||||||
|
|
@ -12478,6 +12478,30 @@ class MusicDatabase:
|
||||||
logger.debug(f"Error adding library history entry: {e}")
|
logger.debug(f"Error adding library history entry: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def get_origin_cleanup_candidates(self):
|
||||||
|
"""Origin-tracked downloads (watchlist/playlist) annotated with the
|
||||||
|
matching library track's play_count, for the Expired Download Cleaner.
|
||||||
|
play_count is 0 when no library track matches the recorded path
|
||||||
|
(orphan history row → treated as not-listened)."""
|
||||||
|
try:
|
||||||
|
conn = self._get_connection()
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute("""
|
||||||
|
SELECT lh.id, lh.origin, lh.origin_context, lh.created_at,
|
||||||
|
lh.file_path, lh.title, lh.artist_name,
|
||||||
|
COALESCE(t.play_count, 0) AS play_count
|
||||||
|
FROM library_history lh
|
||||||
|
LEFT JOIN tracks t ON t.file_path = lh.file_path
|
||||||
|
WHERE lh.event_type = 'download'
|
||||||
|
AND lh.origin IN ('watchlist', 'playlist')
|
||||||
|
""")
|
||||||
|
cols = ['id', 'origin', 'origin_context', 'created_at',
|
||||||
|
'file_path', 'title', 'artist_name', 'play_count']
|
||||||
|
return [dict(zip(cols, row, strict=True)) for row in cursor.fetchall()]
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"Error getting origin cleanup candidates: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
def get_download_origin_entries(self, origin, limit=200, offset=0):
|
def get_download_origin_entries(self, origin, limit=200, offset=0):
|
||||||
"""Downloads triggered by ``origin`` ('watchlist' / 'playlist'),
|
"""Downloads triggered by ``origin`` ('watchlist' / 'playlist'),
|
||||||
newest first. Returns (entries, total_count)."""
|
newest first. Returns (entries, total_count)."""
|
||||||
|
|
|
||||||
91
tests/library/test_expired_cleanup.py
Normal file
91
tests/library/test_expired_cleanup.py
Normal file
|
|
@ -0,0 +1,91 @@
|
||||||
|
"""Pure expiry decision for the Expired Download Cleaner."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from core.library.expired_cleanup import (
|
||||||
|
retention_cutoff,
|
||||||
|
is_expired,
|
||||||
|
select_expired,
|
||||||
|
)
|
||||||
|
|
||||||
|
NOW = datetime(2026, 6, 7, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _entry(origin="playlist", days_old=100, play_count=0, protected=False, eid=1):
|
||||||
|
return {
|
||||||
|
"id": eid, "origin": origin, "play_count": play_count, "protected": protected,
|
||||||
|
"created_at": (NOW - timedelta(days=days_old)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _check(entry, wl="off", pl="2mo", min_plays=2):
|
||||||
|
return is_expired(entry, watchlist_retention=wl, playlist_retention=pl,
|
||||||
|
min_plays=min_plays, now=NOW)
|
||||||
|
|
||||||
|
|
||||||
|
# ── retention windows ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_retention_cutoff_maps_durations():
|
||||||
|
assert retention_cutoff("2mo", NOW) == NOW - timedelta(days=60)
|
||||||
|
assert retention_cutoff("1w", NOW) == NOW - timedelta(days=7)
|
||||||
|
assert retention_cutoff("off", NOW) is None
|
||||||
|
assert retention_cutoff(None, NOW) is None
|
||||||
|
assert retention_cutoff("bogus", NOW) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_expired_only_past_window():
|
||||||
|
assert _check(_entry(days_old=70), pl="2mo") is True # 70 > 60d
|
||||||
|
assert _check(_entry(days_old=50), pl="2mo") is False # 50 < 60d
|
||||||
|
|
||||||
|
|
||||||
|
def test_off_retention_never_expires():
|
||||||
|
assert _check(_entry(origin="watchlist", days_old=999), wl="off") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_origin_uses_its_own_window():
|
||||||
|
wl = _entry(origin="watchlist", days_old=30)
|
||||||
|
# watchlist=1w (expired at 30d), playlist=off
|
||||||
|
assert is_expired(wl, watchlist_retention="1w", playlist_retention="off",
|
||||||
|
min_plays=2, now=NOW) is True
|
||||||
|
pl = _entry(origin="playlist", days_old=30)
|
||||||
|
assert is_expired(pl, watchlist_retention="1w", playlist_retention="off",
|
||||||
|
min_plays=2, now=NOW) is False # playlist off
|
||||||
|
|
||||||
|
|
||||||
|
# ── the keep guards ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_protected_kept_even_if_old():
|
||||||
|
assert _check(_entry(days_old=999, protected=True), pl="1w") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_played_more_than_once_kept():
|
||||||
|
assert _check(_entry(days_old=999, play_count=2), pl="1w", min_plays=2) is False
|
||||||
|
assert _check(_entry(days_old=999, play_count=1), pl="1w", min_plays=2) is True # one play = deletable
|
||||||
|
assert _check(_entry(days_old=999, play_count=0), pl="1w", min_plays=2) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_min_plays_threshold_configurable():
|
||||||
|
e = _entry(days_old=999, play_count=1)
|
||||||
|
assert _check(e, pl="1w", min_plays=1) is False # keep-if-played-at-least-1
|
||||||
|
assert _check(e, pl="1w", min_plays=3) is True # needs 3 plays to keep
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_age_never_deleted():
|
||||||
|
e = _entry(days_old=999)
|
||||||
|
e["created_at"] = "garbage"
|
||||||
|
assert _check(e, pl="1w") is False
|
||||||
|
|
||||||
|
|
||||||
|
# ── select_expired ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_select_expired_filters():
|
||||||
|
entries = [
|
||||||
|
_entry(eid=1, days_old=70, play_count=0), # expired
|
||||||
|
_entry(eid=2, days_old=70, play_count=5), # listened → keep
|
||||||
|
_entry(eid=3, days_old=70, protected=True), # mirrored → keep
|
||||||
|
_entry(eid=4, days_old=10), # too new → keep
|
||||||
|
]
|
||||||
|
out = select_expired(entries, watchlist_retention="off", playlist_retention="2mo")
|
||||||
|
assert [e["id"] for e in out] == [1]
|
||||||
137
tests/test_expired_download_cleaner.py
Normal file
137
tests/test_expired_download_cleaner.py
Normal file
|
|
@ -0,0 +1,137 @@
|
||||||
|
"""Expired Download Cleaner job: scan protection + findings vs auto-delete,
|
||||||
|
and the shared delete helper.
|
||||||
|
|
||||||
|
The pure expiry logic is tested in tests/library/test_expired_cleanup.py; this
|
||||||
|
covers the job's fact-gathering (play_count, active-mirror/watch protection)
|
||||||
|
and the two modes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.repair_jobs.expired_download_cleaner import (
|
||||||
|
ExpiredDownloadCleanerJob,
|
||||||
|
delete_origin_download,
|
||||||
|
)
|
||||||
|
|
||||||
|
OLD = (datetime.now(timezone.utc) - timedelta(days=120)).strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
NEW = (datetime.now(timezone.utc) - timedelta(days=2)).strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
|
||||||
|
|
||||||
|
class _DB:
|
||||||
|
def __init__(self, candidates, mirrored=None, watched=None):
|
||||||
|
self._candidates = candidates
|
||||||
|
self._mirrored = mirrored or []
|
||||||
|
self._watched = watched or []
|
||||||
|
self.deleted_paths = []
|
||||||
|
self.deleted_history = []
|
||||||
|
|
||||||
|
def get_origin_cleanup_candidates(self):
|
||||||
|
return [dict(c) for c in self._candidates]
|
||||||
|
|
||||||
|
def get_mirrored_playlists(self, profile_id=1):
|
||||||
|
return [{'name': n} for n in self._mirrored]
|
||||||
|
|
||||||
|
def get_watchlist_artists(self, profile_id=1):
|
||||||
|
return [SimpleNamespace(artist_name=n) for n in self._watched]
|
||||||
|
|
||||||
|
def delete_track_by_file_path(self, p):
|
||||||
|
self.deleted_paths.append(p)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
def delete_library_history_rows(self, ids):
|
||||||
|
self.deleted_history.extend(ids)
|
||||||
|
return len(ids)
|
||||||
|
|
||||||
|
|
||||||
|
def _ctx(db, settings, findings):
|
||||||
|
return SimpleNamespace(
|
||||||
|
db=db,
|
||||||
|
config_manager=SimpleNamespace(get=lambda k, d=None: settings if k.endswith('.settings') else d),
|
||||||
|
check_stop=lambda: False, wait_if_paused=lambda: False,
|
||||||
|
update_progress=lambda *a, **k: None, report_progress=lambda *a, **k: None,
|
||||||
|
create_finding=lambda **kw: (findings.append(kw) or True),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _cand(eid, origin="playlist", created=OLD, play_count=0, ctx="Some Playlist", path=None):
|
||||||
|
return {"id": eid, "origin": origin, "origin_context": ctx, "created_at": created,
|
||||||
|
"file_path": path or f"/music/{eid}.flac", "title": f"T{eid}",
|
||||||
|
"artist_name": "Artist", "play_count": play_count}
|
||||||
|
|
||||||
|
|
||||||
|
# ── scan: findings mode + protections ────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_scan_noop_when_both_retentions_off():
|
||||||
|
db = _DB([_cand(1)])
|
||||||
|
findings = []
|
||||||
|
res = ExpiredDownloadCleanerJob().scan(_ctx(db, {}, findings)) # defaults: both off
|
||||||
|
assert res.findings_created == 0 and findings == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_creates_findings_for_expired():
|
||||||
|
db = _DB([
|
||||||
|
_cand(1, created=OLD, play_count=0), # expired
|
||||||
|
_cand(2, created=NEW, play_count=0), # too new
|
||||||
|
_cand(3, created=OLD, play_count=5), # listened → keep
|
||||||
|
])
|
||||||
|
findings = []
|
||||||
|
res = ExpiredDownloadCleanerJob().scan(_ctx(
|
||||||
|
db, {'playlist_retention': '2mo', 'keep_if_played_at_least': 2}, findings))
|
||||||
|
assert res.findings_created == 1
|
||||||
|
assert findings[0]['details']['history_id'] == 1
|
||||||
|
assert findings[0]['finding_type'] == 'expired_download'
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_protects_actively_mirrored_playlist():
|
||||||
|
db = _DB([_cand(1, origin="playlist", ctx="My Mix", created=OLD)],
|
||||||
|
mirrored=["My Mix"])
|
||||||
|
findings = []
|
||||||
|
ExpiredDownloadCleanerJob().scan(_ctx(db, {'playlist_retention': '1w'}, findings))
|
||||||
|
assert findings == [] # still mirrored → protected
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_protects_watched_artist():
|
||||||
|
db = _DB([_cand(1, origin="watchlist", ctx="Drake", created=OLD)],
|
||||||
|
watched=["Drake"])
|
||||||
|
findings = []
|
||||||
|
ExpiredDownloadCleanerJob().scan(_ctx(db, {'watchlist_retention': '1w'}, findings))
|
||||||
|
assert findings == [] # still watched → protected
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_auto_delete_mode():
|
||||||
|
db = _DB([_cand(1, created=OLD, path="/music/x.flac")])
|
||||||
|
findings = []
|
||||||
|
res = ExpiredDownloadCleanerJob().scan(_ctx(
|
||||||
|
db, {'playlist_retention': '2mo', 'auto_delete': True}, findings))
|
||||||
|
assert findings == [] # no findings in auto mode
|
||||||
|
assert res.auto_fixed == 1
|
||||||
|
assert 1 in db.deleted_history # history row removed
|
||||||
|
assert "/music/x.flac" in db.deleted_paths # track row removed
|
||||||
|
|
||||||
|
|
||||||
|
# ── delete helper ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_delete_origin_download_missing_file(tmp_path):
|
||||||
|
# File doesn't exist → still cleans up the history row (orphan), no error.
|
||||||
|
db = _DB([])
|
||||||
|
entry = {"id": 9, "file_path": str(tmp_path / "gone.flac")}
|
||||||
|
cfg = SimpleNamespace(get=lambda k, d=None: d)
|
||||||
|
res = delete_origin_download(db, entry, cfg)
|
||||||
|
assert res["error"] is None and res["file_deleted"] is False
|
||||||
|
assert db.deleted_history == [9]
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_origin_download_removes_real_file(tmp_path):
|
||||||
|
f = tmp_path / "song.flac"; f.write_bytes(b"x")
|
||||||
|
db = _DB([])
|
||||||
|
entry = {"id": 5, "file_path": str(f)}
|
||||||
|
cfg = SimpleNamespace(get=lambda k, d=None: d)
|
||||||
|
res = delete_origin_download(db, entry, cfg)
|
||||||
|
assert res["file_deleted"] is True and not f.exists()
|
||||||
|
assert db.deleted_history == [5]
|
||||||
|
|
@ -2734,7 +2734,7 @@ async function loadRepairFindings() {
|
||||||
duplicate_tracks: 'Duplicate', incomplete_album: 'Incomplete',
|
duplicate_tracks: 'Duplicate', incomplete_album: 'Incomplete',
|
||||||
path_mismatch: 'Path Mismatch', metadata_gap: 'Missing Metadata',
|
path_mismatch: 'Path Mismatch', metadata_gap: 'Missing Metadata',
|
||||||
missing_cover_art: 'Missing Art', track_number_mismatch: 'Track Number',
|
missing_cover_art: 'Missing Art', track_number_mismatch: 'Track Number',
|
||||||
missing_lyrics: 'Missing Lyrics',
|
missing_lyrics: 'Missing Lyrics', expired_download: 'Expired',
|
||||||
missing_lossy_copy: 'No Lossy Copy', library_retag: 'Re-tag'
|
missing_lossy_copy: 'No Lossy Copy', library_retag: 'Re-tag'
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -2745,6 +2745,7 @@ async function loadRepairFindings() {
|
||||||
track_number_mismatch: 'Fix',
|
track_number_mismatch: 'Fix',
|
||||||
missing_cover_art: 'Apply Art',
|
missing_cover_art: 'Apply Art',
|
||||||
missing_lyrics: 'Apply Lyrics',
|
missing_lyrics: 'Apply Lyrics',
|
||||||
|
expired_download: 'Delete',
|
||||||
metadata_gap: 'Apply',
|
metadata_gap: 'Apply',
|
||||||
duplicate_tracks: 'Keep Best',
|
duplicate_tracks: 'Keep Best',
|
||||||
incomplete_album: 'Auto-Fill',
|
incomplete_album: 'Auto-Fill',
|
||||||
|
|
@ -2762,6 +2763,7 @@ async function loadRepairFindings() {
|
||||||
already_gone: 'Already Gone', fixed_track_number: 'Track # Fixed',
|
already_gone: 'Already Gone', fixed_track_number: 'Track # Fixed',
|
||||||
applied_cover_art: 'Art Applied', applied_metadata: 'Metadata Applied',
|
applied_cover_art: 'Art Applied', applied_metadata: 'Metadata Applied',
|
||||||
applied_lyrics: 'Lyrics Applied',
|
applied_lyrics: 'Lyrics Applied',
|
||||||
|
deleted_expired: 'Deleted',
|
||||||
removed_duplicates: 'Duplicates Removed',
|
removed_duplicates: 'Duplicates Removed',
|
||||||
};
|
};
|
||||||
let statusBadge = '';
|
let statusBadge = '';
|
||||||
|
|
@ -3136,6 +3138,13 @@ function _renderFindingDetail(f) {
|
||||||
if (d.album_title) rows.push(['Album', d.album_title]);
|
if (d.album_title) rows.push(['Album', d.album_title]);
|
||||||
return _gridRows(rows);
|
return _gridRows(rows);
|
||||||
|
|
||||||
|
case 'expired_download':
|
||||||
|
if (d.title) rows.push(['Track', d.title]);
|
||||||
|
if (d.artist) rows.push(['Artist', d.artist]);
|
||||||
|
if (d.origin) rows.push(['Source', `${d.origin}${d.origin_context ? ' — ' + d.origin_context : ''}`]);
|
||||||
|
if (d.file_path) rows.push(['File', d.file_path.split(/[\\/]/).pop()]);
|
||||||
|
return _gridRows(rows);
|
||||||
|
|
||||||
case 'track_number_mismatch':
|
case 'track_number_mismatch':
|
||||||
if (d.album_title) rows.push(['Album', d.album_title]);
|
if (d.album_title) rows.push(['Album', d.album_title]);
|
||||||
if (d.artist_name) rows.push(['Artist', d.artist_name]);
|
if (d.artist_name) rows.push(['Artist', d.artist_name]);
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue