soulsync/core/repair_jobs/acoustid_scanner.py
BoulderBadgeDad 3a92571c71 downloads: stop AcoustID scan duplicating history rows / leaving verified tracks 'unverified' (#934)
the AcoustID scanner matched library_history rows by EXACT file_path, but that path is
frozen at import time while the file moves afterward (media-server import / reorganize) —
so tracks.file_path (what the scan reads) no longer equals it. two failures resulted, both
introduced in 37ea6604: verified status never reached the history row (verified tracks kept
showing 'unverified'), and a fresh acoustid_scan row was INSERTed every run (5551 rows for
3675 songs).

- new pure, tested matcher (core/downloads/history_match.py): exact path → filename guarded
  by title; prefers a real download row over a synthetic scan row.
- _persist_status now HEALS the matched row's path + status (so future scans match cleanly),
  DELETES synthetic acoustid_scan duplicates by exact path (collision-free, never a real row),
  and inserts only when the file genuinely has no row.
- a full AcoustID job now self-cleans existing duplicates — no destructive bulk migration.

8 matcher + 4 real-DB heal/dedup/insert tests; existing scanner tests updated to the new
seam (heal vs insert). 1076 acoustid/verification/download tests green.
2026-06-27 09:52:39 -07:00

560 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""AcoustID Scanner Job — fingerprints library tracks to detect wrong downloads.
Scans the entire library (not just Transfer) by resolving DB file paths to
actual files on disk. Creates actionable findings that can be fixed:
- 'retag': Update DB metadata to match what the file actually is
- 'redownload': Add the expected track to wishlist and delete the wrong file
- 'delete': Remove the wrong file and its DB record
"""
import os
import re
from difflib import SequenceMatcher
from typing import Optional
from core.repair_jobs import register_job
from core.repair_jobs.base import JobContext, JobResult, RepairJob
from utils.logging_config import get_logger
from core.matching.audio_verification import evaluate, Decision
from core.matching.acoustid_candidates import duration_mismatches_strongly
from core.acoustid_verification import _resolve_expected_artist_aliases
logger = get_logger("repair_job.acoustid")
AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'}
@register_job
class AcoustIDScannerJob(RepairJob):
job_id = 'acoustid_scanner'
display_name = 'AcoustID Scanner'
description = 'Fingerprints library tracks to detect wrong downloads'
help_text = (
'Scans your music library by fingerprinting audio files and comparing '
'them against the AcoustID database. Detects cases where the wrong song '
'was downloaded — even if the filename and tags look correct.\n\n'
'When a mismatch is found, you can:\n'
'• Retag — update the DB record to match the actual audio content\n'
'• Redownload — add the correct track to your wishlist and remove the wrong file\n'
'• Delete — remove the wrong file entirely\n\n'
'The job processes tracks in batches with checkpointing so it resumes '
'where it left off across runs. Requires an AcoustID API key (Settings).\n\n'
'Settings:\n'
'- Fingerprint Threshold: Minimum AcoustID match confidence (0.01.0)\n'
'- Title Similarity: How closely the identified title must match\n'
'- Artist Similarity: How closely the identified artist must match\n'
'- Batch Size: Tracks per scan run (checkpoint saved between batches)'
)
icon = 'repair-icon-acoustid'
default_enabled = True
default_interval_hours = 24
default_settings = {
'fingerprint_threshold': 0.80,
'title_similarity': 0.70,
'artist_similarity': 0.60,
'batch_size': 200,
}
auto_fix = False # User chooses fix action per finding
def scan(self, context: JobContext) -> JobResult:
result = JobResult()
settings = self._get_settings(context)
fp_threshold = settings.get('fingerprint_threshold', 0.80)
title_threshold = settings.get('title_similarity', 0.70)
artist_threshold = settings.get('artist_similarity', 0.60)
batch_size = settings.get('batch_size', 200)
# Get AcoustID client
acoustid_client = context.acoustid_client
if not acoustid_client:
try:
from core.acoustid_client import AcoustIDClient
acoustid_client = AcoustIDClient()
except Exception as e:
logger.warning("AcoustID client not available: %s", e)
return result
# Load all library tracks from DB with their file paths
db_tracks = self._load_db_tracks(context)
if not db_tracks:
logger.info("No library tracks with file paths found")
return result
# Read checkpoint (last processed track ID) to resume from
checkpoint_id = None
if context.config_manager:
checkpoint_id = context.config_manager.get(
f'repair.jobs.{self.job_id}.checkpoint_id', None
)
if checkpoint_id is not None:
checkpoint_id = str(checkpoint_id)
# Build ordered list of (track_id, info) sorted by ID for deterministic order
track_list = sorted(db_tracks.items(), key=lambda x: str(x[0]))
# Skip past checkpoint if resuming
if checkpoint_id is not None:
original_len = len(track_list)
track_list = [(tid, info) for tid, info in track_list if str(tid) > checkpoint_id]
if len(track_list) < original_len:
logger.info("Resuming AcoustID scan from checkpoint ID %s (%d tracks remaining)",
checkpoint_id, len(track_list))
total = len(track_list)
if context.report_progress:
context.report_progress(phase=f'Scanning {total} library tracks...', total=total)
if context.update_progress:
context.update_progress(0, total)
batch_count = 0
for i, (track_id, track_info) in enumerate(track_list):
if context.check_stop():
self._save_checkpoint_id(context, track_id)
return result
if i % 10 == 0 and context.wait_if_paused():
self._save_checkpoint_id(context, track_id)
return result
# Resolve the DB path to an actual file on disk
file_path = track_info.get('file_path', '')
resolved = self._resolve_path(file_path, context)
if not resolved:
result.skipped += 1
continue
result.scanned += 1
batch_count += 1
fname = os.path.basename(resolved)
if context.report_progress:
context.report_progress(
scanned=i + 1, total=total,
phase=f'Fingerprinting {i + 1} / {total}',
log_line=f'Scanning: {fname}',
log_type='info'
)
try:
self._scan_file(
resolved, track_id, track_info, acoustid_client, context, result,
fp_threshold, title_threshold, artist_threshold
)
except Exception as e:
logger.debug("Error scanning %s: %s", fname, e)
result.errors += 1
# Rate limit: pause between batches to avoid hammering AcoustID API
if batch_count >= batch_size:
batch_count = 0
self._save_checkpoint_id(context, track_id)
if context.sleep_or_stop(2):
return result
if context.update_progress and (i + 1) % 10 == 0:
context.update_progress(i + 1, total)
# Clear checkpoint on full completion
self._save_checkpoint_id(context, None)
if context.update_progress:
context.update_progress(total, total)
logger.info("AcoustID scan: %d scanned, %d skipped, %d mismatches, %d errors",
result.scanned, result.skipped, result.findings_created, result.errors)
return result
def _scan_file(self, fpath, track_id, expected, acoustid_client, context, result,
fp_threshold, title_threshold, artist_threshold):
"""Fingerprint a single file and check for mismatches."""
fname = os.path.basename(fpath)
# Fingerprint the file
try:
fp_result = acoustid_client.fingerprint_and_lookup(fpath)
except Exception as e:
logger.debug("Fingerprint failed for %s: %s", fname, e)
result.errors += 1
if context.report_progress:
context.report_progress(log_line=f'Error: {fname}{e}', log_type='error')
return
if not fp_result or not fp_result.get('recordings'):
if context.report_progress:
context.report_progress(log_line=f'No match: {fname}', log_type='skip')
return
best_score = fp_result.get('best_score', 0)
if best_score < fp_threshold:
return
best_recording = fp_result['recordings'][0]
aid_title = best_recording.get('title', '')
aid_artist = best_recording.get('artist', '')
if not aid_title:
return
# Resolve which artist value to compare against, in priority order:
# 1. DB `track_artist` (per-track, manually curated or scanner-
# populated) — trust it when populated. Respects user edits
# from the enhanced library view.
# 2. File's ARTIST tag — ground truth for what's on disk.
# Catches legacy compilation tracks where `track_artist`
# column is NULL because they were downloaded before that
# column existed; the file itself has the correct per-
# track artist (Tidal/Spotify/Deezer all write it).
# 3. Album artist — final fallback for files without proper
# ARTIST tags AND no DB track_artist.
track_artist = (expected.get('track_artist') or '').strip()
if track_artist:
expected_artist = track_artist
else:
file_artist = None
try:
from core.tag_writer import read_file_tags
file_tags = read_file_tags(fpath)
file_artist = (file_tags.get('artist') or '').strip() or None
except Exception as e:
logger.debug("file-tag artist read failed for %s: %s", fname, e)
expected_artist = (
file_artist
or (expected.get('album_artist') or '').strip()
or expected['artist']
)
# Verification status from the embedded SOULSYNC_VERIFICATION tag.
# Only a human decision short-circuits the scan: the user explicitly
# confirmed the file via the review queue. Everything else (verified /
# unverified / force_imported / untagged) is re-checked; force_imported
# mismatches are reported as informational below since a mismatch
# there is EXPECTED (the user accepted the best candidate).
file_verif_status = None
try:
from core.tag_writer import read_file_tags as _rft
file_verif_status = (_rft(fpath) or {}).get('verification_status')
except Exception: # noqa: S110 — verification tag is optional context; None is fine
pass
if file_verif_status == 'human_verified':
# The user explicitly confirmed this file via the review queue —
# never second-guess a human decision.
if context.report_progress:
context.report_progress(
log_line=f'Skipped (human-verified): {fname}', log_type='skip')
return
# Fingerprint-collision guard: when the TOP recording's length is wildly
# different from the file, the fingerprint hit is a hash collision (the
# 17-min mashup → 5-min track case), not a real match — skip BEFORE any
# title/artist/version analysis so it can't surface as a false finding.
try:
file_duration_s = (expected.get('duration_ms') or 0) / 1000.0
except Exception:
file_duration_s = 0.0
cand_duration_s = best_recording.get('duration') or best_recording.get('length')
if file_duration_s and duration_mismatches_strongly(file_duration_s, cand_duration_s):
if context.report_progress:
context.report_progress(
log_line=(f'Skipped (duration mismatch suggests fingerprint '
f'collision): {fname}'),
log_type='skip')
return
# Decision via the shared verification core — identical logic to import-
# time verification (alias-aware artist match + cross-script SKIP), so the
# scan no longer false-flags correct cross-script tracks. Only a FAIL
# produces a "Wrong download" finding.
_alias_cache = {}
def _aliases():
if 'v' not in _alias_cache:
try:
_alias_cache['v'] = _resolve_expected_artist_aliases(expected_artist)
except Exception:
_alias_cache['v'] = []
return _alias_cache['v']
outcome = evaluate(
expected['title'], expected_artist, fp_result['recordings'],
fingerprint_score=best_score,
aliases_provider=_aliases,
)
# Persist the scan outcome so it feeds the same review pipeline as
# import-time verification: PASS backfills 'verified' on untagged or
# previously-unverified files; SKIP (ambiguous / cross-script / no
# hard confirmation) marks untagged files 'unverified' so they surface
# in the Downloads-page review queue. force_imported is never blessed
# here (normalize() strips version words, so an instrumental can PASS
# the title check) and 'verified' is never downgraded by a SKIP (the
# import-time check ran with richer candidate metadata). FAIL keeps
# the finding flow below.
new_status = file_verif_status
if outcome.decision == Decision.PASS and file_verif_status in (None, '', 'unverified'):
new_status = 'verified'
elif outcome.decision == Decision.SKIP and not file_verif_status:
new_status = 'unverified'
if new_status:
self._persist_status(
context, track_id, fpath,
(expected.get('file_path') or '').strip() or None,
new_status, write_tag=(new_status != file_verif_status),
expected=expected)
if outcome.decision != Decision.FAIL:
if context.report_progress:
context.report_progress(
log_line=f'OK ({outcome.decision.value}): {fname}{outcome.reason}',
log_type='ok',
)
return
title_sim = outcome.title_sim
artist_sim = outcome.artist_sim
matched_title = outcome.matched_title or aid_title
matched_artist = outcome.matched_artist or aid_artist
# Mismatch (FAIL) — create finding.
if context.report_progress:
context.report_progress(
log_line=f'Mismatch: {fname} — expected "{expected["title"]}", got "{matched_title}"',
log_type='error'
)
if context.create_finding:
_is_force = file_verif_status == 'force_imported'
severity = 'info' if _is_force else ('warning' if best_score >= 0.90 else 'info')
_title = (
f'Force-imported (fallback): "{expected["title"]}" is actually "{matched_title}"'
if _is_force else
f'Wrong download: "{expected["title"]}" is actually "{matched_title}"'
)
inserted = context.create_finding(
job_id=self.job_id,
finding_type='acoustid_mismatch',
severity=severity,
entity_type='track',
entity_id=str(track_id),
file_path=fpath,
title=_title,
description=(
f'Expected "{expected["title"]}" by {expected_artist}, '
f'but audio fingerprint matches "{matched_title}" by {matched_artist} '
f'(fingerprint: {best_score:.0%}, title match: {title_sim:.0%}, '
f'artist match: {artist_sim:.0%})'
),
details={
'expected_title': expected['title'],
'expected_artist': expected_artist,
'acoustid_title': matched_title,
'acoustid_artist': matched_artist,
'fingerprint_score': round(best_score, 3),
'title_similarity': round(title_sim, 3),
'artist_similarity': round(artist_sim, 3),
'album_thumb_url': expected.get('album_thumb_url'),
'artist_thumb_url': expected.get('artist_thumb_url'),
'album_title': expected.get('album_title', ''),
'track_number': expected.get('track_number'),
'force_imported': file_verif_status == 'force_imported',
}
)
if inserted:
result.findings_created += 1
else:
result.findings_skipped_dedup += 1
def _persist_status(self, context, track_id, fpath, db_path, status, write_tag,
expected=None):
"""Persist a verification status to the file tag (durable, travels with
the file), the tracks row (UI cache) and any library_history rows for
this file (feeds the Unverified review queue on the Downloads page).
``db_path`` is the unresolved DB-side path — history rows may store
either form, so both are matched.
Files SoulSync never downloaded have no history row at all — for an
'unverified' outcome one is inserted (download_source 'acoustid_scan')
so EVERY scan-flagged file lands in the review queue, not just past
downloads. Re-scans then match this row via file_path (no duplicates).
"""
if not status:
return
if write_tag:
try:
from core.tag_writer import write_verification_status
write_verification_status(fpath, status)
except Exception as e:
logger.debug("verification tag write failed for %s: %s", fpath, e)
try:
conn = context.db._get_connection()
cur = conn.cursor()
cur.execute(
"UPDATE tracks SET verification_status = ? WHERE id = ?",
(status, track_id))
exp = expected or {}
# Find the canonical history row for this file. The stored path is frozen at
# import time while the file has since moved (media-server import / reorganize),
# so an exact-path match alone misses it — then the status never lands and a
# duplicate row gets inserted every scan (#934). Match exact path first, then
# filename guarded by title, and HEAL the row's path so future scans match cleanly.
from core.downloads.history_match import pick_history_row, like_filename_filter
current = fpath or db_path
basename = os.path.basename(current) if current else ''
clauses, params = [], []
for p in {p for p in (fpath, db_path) if p}:
clauses.append("file_path = ?")
params.append(p)
if basename:
clauses.append("file_path LIKE ? ESCAPE '\\'")
params.append(like_filename_filter(basename))
row_id = None
if clauses:
cur.execute(
"SELECT id, file_path, title, download_source FROM library_history WHERE "
+ " OR ".join(clauses),
params)
row_id = pick_history_row(
cur.fetchall(),
current_paths=(fpath, db_path),
basename=basename, title=exp.get('title') or '')
if row_id is not None:
cur.execute(
"UPDATE library_history SET verification_status = ?, file_path = ? WHERE id = ?",
(status, current, row_id))
# Drop synthetic scan-created duplicates for this exact file (the #934
# leftovers). Exact path → collision-free; never touches a real download row.
cur.execute(
"DELETE FROM library_history WHERE id != ? AND download_source = 'acoustid_scan' "
"AND file_path = ?",
(row_id, current))
elif status == 'unverified':
cur.execute(
"""INSERT INTO library_history
(event_type, title, artist_name, album_name, file_path,
thumb_url, download_source, verification_status)
VALUES ('download', ?, ?, ?, ?, ?, 'acoustid_scan', ?)""",
(exp.get('title') or os.path.basename(fpath),
exp.get('artist') or None,
exp.get('album_title') or None,
db_path or fpath,
exp.get('album_thumb_url') or None,
status))
getattr(conn, 'commit', lambda: None)()
except Exception as e:
logger.debug("verification_status persist failed for %s: %s", track_id, e)
def _load_db_tracks(self, context: JobContext) -> dict:
"""Load all tracks from DB keyed by track ID."""
tracks = {}
conn = None
try:
conn = context.db._get_connection()
cursor = conn.cursor()
# Discord report (Skowl): compilation albums like "High Tea
# Music: Vol 1" have a different artist per track but the
# `tracks.artist_id` foreign key points at the ALBUM artist
# (curator / label-name applied to every track). AcoustID
# returns the actual per-track artist → 12% similarity →
# Wrong Song flag. Fix: prefer `tracks.track_artist` (the
# per-track artist, populated by every server-scan + auto-
# import path when different from album artist) and fall
# back to the album artist only when the per-track column
# is NULL or empty (legacy rows / single-artist albums).
# Load `track_artist` (raw, may be empty) AND `album_artist`
# separately so `_scan_file` can tell the difference between
# 'DB has a curated per-track value' and 'DB fell back to
# album artist'. The COALESCE'd `artist` field is kept as a
# convenience for the existing `expected['artist']` consumers
# that want a single resolved value, but the resolution
# priority that actually drives the comparison is reproduced
# in `_scan_file`: track_artist → file tag → album_artist.
cursor.execute("""
SELECT t.id, t.title,
COALESCE(NULLIF(t.track_artist, ''), ar.name) AS artist,
t.file_path, t.track_number,
al.title AS album_title, al.thumb_url, ar.thumb_url,
NULLIF(t.track_artist, '') AS track_artist,
ar.name AS album_artist,
t.duration
FROM tracks t
LEFT JOIN artists ar ON ar.id = t.artist_id
LEFT JOIN albums al ON al.id = t.album_id
WHERE t.file_path IS NOT NULL AND t.file_path != ''
AND t.title IS NOT NULL AND t.title != ''
""")
for row in cursor.fetchall():
track_id = row[0]
if track_id is None:
logger.warning(
"Skipping track row with null ID while loading AcoustID scan candidates: %s",
row[3] or "<unknown file>",
)
continue
track_id = str(track_id)
tracks[track_id] = {
'title': row[1] or '',
'artist': row[2] or '',
'file_path': row[3] or '',
'track_number': row[4],
'album_title': row[5] or '',
'album_thumb_url': row[6] or None,
'artist_thumb_url': row[7] or None,
'track_artist': row[8] or '', # raw (may be empty)
'album_artist': row[9] or '',
# Duration in MS (DB stores ms). Used by the
# duration-mismatch guard to spot fingerprint
# collisions where the matched recording is a
# totally different length.
'duration_ms': row[10] or 0,
}
except Exception as e:
logger.error("Error loading tracks from DB: %s", e)
finally:
if conn:
conn.close()
return tracks
def _resolve_path(self, file_path, context):
"""Resolve a DB file path to an actual file on disk."""
if not file_path:
return None
if os.path.exists(file_path):
return file_path
# Use the shared library-path resolver — picks up
# library.music_paths and Plex library locations too.
from core.library.path_resolver import resolve_library_file_path
return resolve_library_file_path(
file_path,
transfer_folder=context.transfer_folder,
config_manager=context.config_manager,
)
def _save_checkpoint_id(self, context: JobContext, track_id):
"""Save or clear the scan checkpoint by track ID."""
if context.config_manager:
context.config_manager.set(
f'repair.jobs.{self.job_id}.checkpoint_id', track_id
)
def _get_settings(self, context: JobContext) -> dict:
if not context.config_manager:
return self.default_settings.copy()
cfg = context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {})
merged = self.default_settings.copy()
merged.update(cfg)
return merged
def estimate_scope(self, context: JobContext) -> int:
conn = None
try:
conn = context.db._get_connection()
cursor = conn.cursor()
cursor.execute("""
SELECT COUNT(*) FROM tracks
WHERE file_path IS NOT NULL AND file_path != ''
AND title IS NOT NULL AND title != ''
""")
return cursor.fetchone()[0]
except Exception:
return 0
finally:
if conn:
conn.close()