455945e6cb
Changed ignore_cross_album default from True to False. Re-downloads of the same song create separate album entries, so the detector was skipping them. Users who want to keep compilations/greatest-hits intact can toggle it back on. Updated help text to explain when to use this setting.
226 lines
9.3 KiB
Python
226 lines
9.3 KiB
Python
"""Duplicate Track Detector Job — finds potential duplicate tracks in the library."""
|
|
|
|
import re
|
|
from collections import defaultdict
|
|
from difflib import SequenceMatcher
|
|
|
|
from core.repair_jobs import register_job
|
|
from core.repair_jobs.base import JobContext, JobResult, RepairJob
|
|
from utils.logging_config import get_logger
|
|
|
|
logger = get_logger("repair_job.duplicates")
|
|
|
|
|
|
@register_job
|
|
class DuplicateDetectorJob(RepairJob):
|
|
job_id = 'duplicate_detector'
|
|
display_name = 'Duplicate Detector'
|
|
description = 'Finds potential duplicate tracks in your library'
|
|
help_text = (
|
|
'Groups tracks by similar title and artist name using fuzzy matching, then flags '
|
|
'groups where multiple copies exist. This helps you find accidental duplicates '
|
|
'from re-downloads, compilation albums, or similar-titled tracks.\n\n'
|
|
'Each duplicate group is reported as a finding with details about every copy '
|
|
'(file path, format, bitrate) so you can decide which to keep.\n\n'
|
|
'Settings:\n'
|
|
'- Title Similarity: How closely titles must match to be considered duplicates (0.0 - 1.0)\n'
|
|
'- Artist Similarity: How closely artist names must match (0.0 - 1.0)\n'
|
|
'- Ignore Cross-Album: When enabled, tracks on different albums are not flagged as duplicates. '
|
|
'Turn this OFF if you have duplicate downloads filed under different album entries — '
|
|
'this is the most common cause of missed duplicates from re-downloads'
|
|
)
|
|
icon = 'repair-icon-duplicate'
|
|
default_enabled = False
|
|
default_interval_hours = 168
|
|
default_settings = {
|
|
'title_similarity': 0.85,
|
|
'artist_similarity': 0.80,
|
|
'ignore_cross_album': False,
|
|
}
|
|
auto_fix = False
|
|
|
|
def scan(self, context: JobContext) -> JobResult:
|
|
result = JobResult()
|
|
|
|
settings = self._get_settings(context)
|
|
title_threshold = float(settings.get('title_similarity', 0.85))
|
|
artist_threshold = float(settings.get('artist_similarity', 0.80))
|
|
ignore_cross_album = settings.get('ignore_cross_album', True)
|
|
|
|
# Fetch all tracks with artist/album names via JOIN
|
|
tracks = []
|
|
conn = None
|
|
try:
|
|
conn = context.db._get_connection()
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT t.id, t.title, ar.name, al.title, t.file_path,
|
|
t.bitrate, t.duration, al.thumb_url, ar.thumb_url
|
|
FROM tracks t
|
|
LEFT JOIN artists ar ON ar.id = t.artist_id
|
|
LEFT JOIN albums al ON al.id = t.album_id
|
|
WHERE t.title IS NOT NULL AND t.title != ''
|
|
AND t.file_path IS NOT NULL AND t.file_path != ''
|
|
""")
|
|
tracks = cursor.fetchall()
|
|
except Exception as e:
|
|
logger.error("Error fetching tracks from DB: %s", e, exc_info=True)
|
|
result.errors += 1
|
|
return result
|
|
finally:
|
|
if conn:
|
|
conn.close()
|
|
|
|
if not tracks:
|
|
return result
|
|
|
|
total = len(tracks)
|
|
if context.update_progress:
|
|
context.update_progress(0, total)
|
|
|
|
# Group tracks by normalized key for fast comparison
|
|
# Bucket by first 4 chars of normalized title for efficiency
|
|
buckets = defaultdict(list)
|
|
for row in tracks:
|
|
track_id, title, artist_name, album_title, file_path, bitrate, duration, album_thumb, artist_thumb = row
|
|
norm_title = _normalize(title)
|
|
bucket_key = norm_title[:4] if len(norm_title) >= 4 else norm_title
|
|
buckets[bucket_key].append({
|
|
'id': track_id,
|
|
'title': title,
|
|
'norm_title': norm_title,
|
|
'artist': artist_name or '',
|
|
'norm_artist': _normalize(artist_name or ''),
|
|
'album': album_title,
|
|
'file_path': file_path,
|
|
'bitrate': bitrate,
|
|
'duration': duration,
|
|
'album_thumb_url': album_thumb or None,
|
|
'artist_thumb_url': artist_thumb or None,
|
|
})
|
|
|
|
# Find duplicates within each bucket
|
|
found_groups = set() # Track IDs already in a group
|
|
processed = 0
|
|
|
|
if context.report_progress:
|
|
context.report_progress(phase=f'Comparing {total} tracks...', total=total)
|
|
|
|
for bucket_key, bucket_tracks in buckets.items():
|
|
if context.check_stop():
|
|
return result
|
|
|
|
for i, t1 in enumerate(bucket_tracks):
|
|
if context.check_stop():
|
|
return result
|
|
|
|
processed += 1
|
|
result.scanned += 1
|
|
|
|
if context.report_progress and processed % 100 == 0:
|
|
context.report_progress(
|
|
scanned=processed, total=total,
|
|
phase=f'Comparing {processed} / {total}',
|
|
log_line=f'Checking: {t1["title"]} — {t1["artist"]}',
|
|
log_type='info'
|
|
)
|
|
|
|
if t1['id'] in found_groups:
|
|
continue
|
|
|
|
group = [t1]
|
|
|
|
for j in range(i + 1, len(bucket_tracks)):
|
|
t2 = bucket_tracks[j]
|
|
if t2['id'] in found_groups:
|
|
continue
|
|
|
|
# Compare titles
|
|
title_sim = SequenceMatcher(None, t1['norm_title'], t2['norm_title']).ratio()
|
|
if title_sim < title_threshold:
|
|
continue
|
|
|
|
# Compare artists
|
|
artist_sim = SequenceMatcher(None, t1['norm_artist'], t2['norm_artist']).ratio()
|
|
if artist_sim < artist_threshold:
|
|
continue
|
|
|
|
# Skip cross-album duplicates — same song on different albums is intentional
|
|
if ignore_cross_album and t1['album'] and t2['album'] and t1['album'] != t2['album']:
|
|
continue
|
|
|
|
group.append(t2)
|
|
|
|
if len(group) >= 2:
|
|
# Found a duplicate group
|
|
for t in group:
|
|
found_groups.add(t['id'])
|
|
|
|
if context.report_progress:
|
|
context.report_progress(
|
|
log_line=f'Duplicate: {t1["title"]} — {len(group)} copies',
|
|
log_type='skip'
|
|
)
|
|
|
|
if context.create_finding:
|
|
try:
|
|
# Sort group by quality (highest bitrate first)
|
|
group.sort(key=lambda t: (t['bitrate'] or 0), reverse=True)
|
|
|
|
context.create_finding(
|
|
job_id=self.job_id,
|
|
finding_type='duplicate_tracks',
|
|
severity='info',
|
|
entity_type='track',
|
|
entity_id=str(group[0]['id']),
|
|
file_path=group[0]['file_path'],
|
|
title=f'Duplicate: {group[0]["title"]} by {group[0]["artist"]}',
|
|
description=f'{len(group)} copies found with similar title/artist',
|
|
details={
|
|
'tracks': [{
|
|
'id': t['id'],
|
|
'title': t['title'],
|
|
'artist': t['artist'],
|
|
'album': t['album'],
|
|
'file_path': t['file_path'],
|
|
'bitrate': t['bitrate'],
|
|
'duration': t['duration'],
|
|
} for t in group],
|
|
'count': len(group),
|
|
'album_thumb_url': group[0].get('album_thumb_url'),
|
|
'artist_thumb_url': group[0].get('artist_thumb_url'),
|
|
}
|
|
)
|
|
result.findings_created += 1
|
|
except Exception as e:
|
|
logger.debug("Error creating duplicate finding: %s", e)
|
|
result.errors += 1
|
|
|
|
if context.update_progress and processed % 200 == 0:
|
|
context.update_progress(processed, total)
|
|
|
|
if context.update_progress:
|
|
context.update_progress(total, total)
|
|
|
|
logger.info("Duplicate scan: %d tracks checked, %d duplicate groups found",
|
|
result.scanned, result.findings_created)
|
|
return result
|
|
|
|
def _get_settings(self, context: JobContext) -> dict:
|
|
if not context.config_manager:
|
|
return self.default_settings.copy()
|
|
cfg = context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {})
|
|
merged = self.default_settings.copy()
|
|
merged.update(cfg)
|
|
return merged
|
|
|
|
|
|
def _normalize(text: str) -> str:
|
|
"""Normalize text for fuzzy comparison.
|
|
|
|
Keeps parenthetical content (remixes, live, etc.) so that similarity
|
|
thresholds can distinguish 'title' from 'title xxx remix'.
|
|
"""
|
|
t = text.lower()
|
|
t = re.sub(r'[^a-z0-9() ]', '', t)
|
|
return t.strip()
|