cf5461f2f1
`_create_finding` silently dedup-skipped re-discovered issues but the caller incremented `findings_created` regardless. So a re-scan that found the same issues as a prior scan reported 364 findings in the badge while 0 NEW pending rows hit the db, leaving the findings tab empty. `_create_finding` now returns bool (True on insert, False on dedup-skip / db error). All 16 repair jobs updated to only increment `findings_created` on True. Added `findings_skipped_dedup` counter surfaced in scan log: "Done: X scanned, 0 fixed, 0 findings (363 already existed), 0 errors". Also fixed a missing `job_id` kwarg in album_tag_consistency that was silently breaking finding creation for that scan.
391 lines
17 KiB
Python
391 lines
17 KiB
Python
"""Duplicate Track Detector Job — finds potential duplicate tracks in the library."""
|
|
|
|
import os
|
|
import re
|
|
from collections import defaultdict
|
|
from difflib import SequenceMatcher
|
|
|
|
from core.imports.file_ops import _strip_slskd_dedup_suffix
|
|
from core.repair_jobs import register_job
|
|
from core.repair_jobs.base import JobContext, JobResult, RepairJob
|
|
from utils.logging_config import get_logger
|
|
|
|
logger = get_logger("repair_job.duplicates")
|
|
|
|
|
|
@register_job
|
|
class DuplicateDetectorJob(RepairJob):
|
|
job_id = 'duplicate_detector'
|
|
display_name = 'Duplicate Detector'
|
|
description = 'Finds potential duplicate tracks in your library'
|
|
help_text = (
|
|
'Groups tracks by similar title and artist name using fuzzy matching, then flags '
|
|
'groups where multiple copies exist. This helps you find accidental duplicates '
|
|
'from re-downloads, compilation albums, or similar-titled tracks.\n\n'
|
|
'Each duplicate group is reported as a finding with details about every copy '
|
|
'(file path, format, bitrate) so you can decide which to keep.\n\n'
|
|
'Settings:\n'
|
|
'- Title Similarity: How closely titles must match to be considered duplicates (0.0 - 1.0)\n'
|
|
'- Artist Similarity: How closely artist names must match (0.0 - 1.0)\n'
|
|
'- Ignore Cross-Album: When enabled, tracks on different albums are not flagged as duplicates. '
|
|
'Turn this OFF if you have duplicate downloads filed under different album entries — '
|
|
'this is the most common cause of missed duplicates from re-downloads'
|
|
)
|
|
icon = 'repair-icon-duplicate'
|
|
default_enabled = False
|
|
default_interval_hours = 168
|
|
default_settings = {
|
|
'title_similarity': 0.85,
|
|
'artist_similarity': 0.80,
|
|
'ignore_cross_album': False,
|
|
}
|
|
auto_fix = False
|
|
|
|
def scan(self, context: JobContext) -> JobResult:
|
|
result = JobResult()
|
|
|
|
settings = self._get_settings(context)
|
|
title_threshold = float(settings.get('title_similarity', 0.85))
|
|
artist_threshold = float(settings.get('artist_similarity', 0.80))
|
|
ignore_cross_album = settings.get('ignore_cross_album', True)
|
|
|
|
# Respect the global "allow duplicate tracks across albums" setting —
|
|
# if the user explicitly allows duplicates across albums, never flag them
|
|
if context.config_manager and context.config_manager.get('library.allow_duplicate_tracks', False):
|
|
ignore_cross_album = True
|
|
|
|
# Fetch all tracks with artist/album names via JOIN
|
|
tracks = []
|
|
conn = None
|
|
try:
|
|
conn = context.db._get_connection()
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT t.id, t.title, ar.name, al.title, t.file_path,
|
|
t.bitrate, t.duration, al.thumb_url, ar.thumb_url
|
|
FROM tracks t
|
|
LEFT JOIN artists ar ON ar.id = t.artist_id
|
|
LEFT JOIN albums al ON al.id = t.album_id
|
|
WHERE t.title IS NOT NULL AND t.title != ''
|
|
AND t.file_path IS NOT NULL AND t.file_path != ''
|
|
""")
|
|
tracks = cursor.fetchall()
|
|
except Exception as e:
|
|
logger.error("Error fetching tracks from DB: %s", e, exc_info=True)
|
|
result.errors += 1
|
|
return result
|
|
finally:
|
|
if conn:
|
|
conn.close()
|
|
|
|
if not tracks:
|
|
return result
|
|
|
|
total = len(tracks)
|
|
if context.update_progress:
|
|
context.update_progress(0, total)
|
|
|
|
# Group tracks by normalized key for fast comparison
|
|
# Bucket by first 4 chars of normalized title for efficiency
|
|
buckets = defaultdict(list)
|
|
for row in tracks:
|
|
track_id, title, artist_name, album_title, file_path, bitrate, duration, album_thumb, artist_thumb = row
|
|
norm_title = _normalize(title)
|
|
bucket_key = norm_title[:4] if len(norm_title) >= 4 else norm_title
|
|
buckets[bucket_key].append({
|
|
'id': track_id,
|
|
'title': title,
|
|
'norm_title': norm_title,
|
|
'artist': artist_name or '',
|
|
'norm_artist': _normalize(artist_name or ''),
|
|
'album': album_title,
|
|
'file_path': file_path,
|
|
'bitrate': bitrate,
|
|
'duration': duration,
|
|
'album_thumb_url': album_thumb or None,
|
|
'artist_thumb_url': artist_thumb or None,
|
|
})
|
|
|
|
# Find duplicates within each bucket
|
|
found_groups = set() # Track IDs already in a group
|
|
processed_holder = {'count': 0}
|
|
|
|
if context.report_progress:
|
|
context.report_progress(phase=f'Comparing {total} tracks...', total=total)
|
|
|
|
# Pass 1 — bucket by normalized-title prefix (existing behavior).
|
|
for _bucket_key, bucket_tracks in buckets.items():
|
|
if context.check_stop():
|
|
return result
|
|
self._scan_bucket(
|
|
bucket_tracks=bucket_tracks,
|
|
require_metadata_match=True,
|
|
title_threshold=title_threshold,
|
|
artist_threshold=artist_threshold,
|
|
ignore_cross_album=ignore_cross_album,
|
|
found_groups=found_groups,
|
|
processed_holder=processed_holder,
|
|
total=total,
|
|
result=result,
|
|
context=context,
|
|
)
|
|
|
|
# Pass 2 — re-bucket leftover tracks by canonical filename stem
|
|
# (slskd dedup suffix stripped). Catches dupes whose tag metadata
|
|
# disagrees because some copies were never properly tagged after
|
|
# download — e.g. ``Song.flac`` and ``Song_<19-digit-ts>.flac``
|
|
# land in the library with identical filenames sans the slskd
|
|
# dedup tail but get inconsistent ID3 titles from the media-server
|
|
# rescan. Pass-1 buckets them apart by title so they never get
|
|
# compared. Discord-reported scenario: 7 copies of one OST track
|
|
# accumulating in one folder, only 1 caught by the detector.
|
|
filename_buckets = self._build_filename_buckets(
|
|
buckets=buckets,
|
|
found_groups=found_groups,
|
|
)
|
|
for _fname_key, fname_tracks in filename_buckets.items():
|
|
if context.check_stop():
|
|
return result
|
|
# Filename match is itself strong evidence — a shared canonical
|
|
# stem means the files came from the same source download.
|
|
# Drop the metadata gates so dedup orphans get caught even
|
|
# when their tag titles disagree.
|
|
self._scan_bucket(
|
|
bucket_tracks=fname_tracks,
|
|
require_metadata_match=False,
|
|
title_threshold=title_threshold,
|
|
artist_threshold=artist_threshold,
|
|
ignore_cross_album=ignore_cross_album,
|
|
found_groups=found_groups,
|
|
processed_holder=processed_holder,
|
|
total=total,
|
|
result=result,
|
|
context=context,
|
|
)
|
|
|
|
if context.update_progress:
|
|
context.update_progress(total, total)
|
|
|
|
logger.info("Duplicate scan: %d tracks checked, %d duplicate groups found",
|
|
result.scanned, result.findings_created)
|
|
return result
|
|
|
|
def _scan_bucket(
|
|
self,
|
|
*,
|
|
bucket_tracks,
|
|
require_metadata_match,
|
|
title_threshold,
|
|
artist_threshold,
|
|
ignore_cross_album,
|
|
found_groups,
|
|
processed_holder,
|
|
total,
|
|
result,
|
|
context,
|
|
) -> None:
|
|
"""Compare every pair within a bucket; emit duplicate groups.
|
|
|
|
``require_metadata_match`` gates the title / artist similarity
|
|
thresholds and the cross-album guard. Pass ``False`` for buckets
|
|
whose grouping is already strong evidence (e.g. shared canonical
|
|
filename) so that dedup orphans with broken / missing tags still
|
|
get caught.
|
|
"""
|
|
for i, t1 in enumerate(bucket_tracks):
|
|
if context.check_stop():
|
|
return
|
|
|
|
processed_holder['count'] += 1
|
|
result.scanned += 1
|
|
processed = processed_holder['count']
|
|
|
|
if context.report_progress and processed % 100 == 0:
|
|
context.report_progress(
|
|
scanned=processed, total=total,
|
|
phase=f'Comparing {processed} / {total}',
|
|
log_line=f'Checking: {t1["title"]} — {t1["artist"]}',
|
|
log_type='info'
|
|
)
|
|
|
|
if t1['id'] in found_groups:
|
|
continue
|
|
|
|
group = [t1]
|
|
|
|
for j in range(i + 1, len(bucket_tracks)):
|
|
t2 = bucket_tracks[j]
|
|
if t2['id'] in found_groups:
|
|
continue
|
|
|
|
if require_metadata_match:
|
|
title_sim = SequenceMatcher(None, t1['norm_title'], t2['norm_title']).ratio()
|
|
if title_sim < title_threshold:
|
|
continue
|
|
artist_sim = SequenceMatcher(None, t1['norm_artist'], t2['norm_artist']).ratio()
|
|
if artist_sim < artist_threshold:
|
|
continue
|
|
if ignore_cross_album and t1['album'] and t2['album'] and t1['album'] != t2['album']:
|
|
continue
|
|
else:
|
|
# Filename-bucket pass: filename agreement is strong but
|
|
# not infallible — two different songs that happen to
|
|
# share a canonical filename (``Yellow.mp3`` by Coldplay
|
|
# vs by Bob's Album) would get grouped without a sanity
|
|
# check. Require duration agreement (within 3s) when
|
|
# both rows have it; same source download = identical
|
|
# duration. If either side is missing duration data,
|
|
# fall back to a relaxed artist similarity check so we
|
|
# don't blindly group strangers.
|
|
if t1['duration'] and t2['duration']:
|
|
if abs(t1['duration'] - t2['duration']) > 3.0:
|
|
continue
|
|
elif t1['norm_artist'] and t2['norm_artist']:
|
|
artist_sim = SequenceMatcher(None, t1['norm_artist'], t2['norm_artist']).ratio()
|
|
if artist_sim < 0.6:
|
|
continue
|
|
# else: both durations missing AND at least one artist
|
|
# is blank — too little signal, skip to avoid false
|
|
# positives.
|
|
elif not t1['norm_artist'] or not t2['norm_artist']:
|
|
continue
|
|
|
|
if _is_same_physical_file(
|
|
t1['file_path'], t2['file_path'],
|
|
t1['duration'], t2['duration'],
|
|
):
|
|
continue
|
|
|
|
group.append(t2)
|
|
|
|
if len(group) >= 2:
|
|
for t in group:
|
|
found_groups.add(t['id'])
|
|
|
|
if context.report_progress:
|
|
context.report_progress(
|
|
log_line=f'Duplicate: {t1["title"]} — {len(group)} copies',
|
|
log_type='skip'
|
|
)
|
|
|
|
if context.create_finding:
|
|
try:
|
|
group.sort(key=lambda t: (t['bitrate'] or 0), reverse=True)
|
|
inserted = context.create_finding(
|
|
job_id=self.job_id,
|
|
finding_type='duplicate_tracks',
|
|
severity='info',
|
|
entity_type='track',
|
|
entity_id=str(group[0]['id']),
|
|
file_path=group[0]['file_path'],
|
|
title=f'Duplicate: {group[0]["title"]} by {group[0]["artist"]}',
|
|
description=f'{len(group)} copies found with similar title/artist',
|
|
details={
|
|
'tracks': [{
|
|
'id': t['id'],
|
|
'title': t['title'],
|
|
'artist': t['artist'],
|
|
'album': t['album'],
|
|
'file_path': t['file_path'],
|
|
'bitrate': t['bitrate'],
|
|
'duration': t['duration'],
|
|
} for t in group],
|
|
'count': len(group),
|
|
'album_thumb_url': group[0].get('album_thumb_url'),
|
|
'artist_thumb_url': group[0].get('artist_thumb_url'),
|
|
}
|
|
)
|
|
if inserted:
|
|
result.findings_created += 1
|
|
else:
|
|
result.findings_skipped_dedup += 1
|
|
except Exception as e:
|
|
logger.debug("Error creating duplicate finding: %s", e)
|
|
result.errors += 1
|
|
|
|
if context.update_progress and processed_holder['count'] % 200 == 0:
|
|
context.update_progress(processed_holder['count'], total)
|
|
|
|
def _build_filename_buckets(self, *, buckets, found_groups):
|
|
"""Re-bucket all tracks by canonical filename stem.
|
|
|
|
The slskd dedup suffix (``_<19+ digit timestamp>``) is stripped so
|
|
``Song.flac`` and ``Song_639122324339578022.flac`` collapse to the
|
|
same key. Singleton buckets (only one track) are dropped — they
|
|
carry no comparison value.
|
|
"""
|
|
filename_buckets = defaultdict(list)
|
|
for bucket_tracks in buckets.values():
|
|
for track in bucket_tracks:
|
|
if track['id'] in found_groups:
|
|
continue
|
|
fp = track.get('file_path') or ''
|
|
if not fp:
|
|
continue
|
|
basename = os.path.basename(str(fp).replace('\\', '/'))
|
|
stem, ext = os.path.splitext(basename)
|
|
if not stem:
|
|
continue
|
|
canonical = _strip_slskd_dedup_suffix(stem)
|
|
key = (canonical.lower(), ext.lower())
|
|
filename_buckets[key].append(track)
|
|
return {k: v for k, v in filename_buckets.items() if len(v) >= 2}
|
|
|
|
def _get_settings(self, context: JobContext) -> dict:
|
|
if not context.config_manager:
|
|
return self.default_settings.copy()
|
|
cfg = context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {})
|
|
merged = self.default_settings.copy()
|
|
merged.update(cfg)
|
|
return merged
|
|
|
|
|
|
def _normalize(text: str) -> str:
|
|
"""Normalize text for fuzzy comparison.
|
|
|
|
Keeps parenthetical content (remixes, live, etc.) so that similarity
|
|
thresholds can distinguish 'title' from 'title xxx remix'.
|
|
"""
|
|
t = text.lower()
|
|
t = re.sub(r'[^a-z0-9() ]', '', t)
|
|
return t.strip()
|
|
|
|
|
|
def _is_same_physical_file(p1, p2, dur1, dur2) -> bool:
|
|
"""Detect when two DB rows point at the same file mounted at different paths.
|
|
|
|
When a user binds the same host music directory into both SoulSync
|
|
(e.g. ``/app/Transfer``) and a media server like Plex (e.g.
|
|
``/media/Music``), the SoulSync scan and the media-server library
|
|
sync each create a track row pointing at the same physical file
|
|
via different mount paths. The two rows then look like a fuzzy-
|
|
match duplicate to this job.
|
|
|
|
Returns True when:
|
|
- Both paths share the last 3 segments (filename + album + artist
|
|
folder), so they really are the same release on disk;
|
|
- The leading mount-root segments differ, ruling out the case
|
|
where one row is just a re-scan of the other path; and
|
|
- When both rows carry a duration, the durations agree within 1
|
|
second (defensive — different files at parallel paths would
|
|
almost always disagree on duration even slightly).
|
|
"""
|
|
if not p1 or not p2:
|
|
return False
|
|
norm1 = str(p1).replace('\\', '/').rstrip('/')
|
|
norm2 = str(p2).replace('\\', '/').rstrip('/')
|
|
parts1 = [x for x in norm1.split('/') if x]
|
|
parts2 = [x for x in norm2.split('/') if x]
|
|
if len(parts1) < 3 or len(parts2) < 3:
|
|
return False
|
|
tail1 = [s.lower() for s in parts1[-3:]]
|
|
tail2 = [s.lower() for s in parts2[-3:]]
|
|
if tail1 != tail2:
|
|
return False
|
|
# Confirm mount roots actually differ, otherwise we'd skip
|
|
# legitimate duplicates that happen to share the trailing path.
|
|
if parts1[:-3] == parts2[:-3]:
|
|
return False
|
|
if dur1 and dur2 and abs(dur1 - dur2) > 1.0:
|
|
return False
|
|
return True
|