39a07e4bdf
Two bugs kept this job from finding anything useful on a typical library. 1. Wrong Deezer column name. The artists table has a deezer_id column (per music_database.py:1986), but the job looked for deezer_artist_id in both _scan_artist (line 132) and _get_library_artists (line 345). For Deezer-primary users, this meant the Deezer ID never made it into the source_ids map, so get_artist_discography fell back to artist- name-only search — slower and less accurate than an ID lookup. 2. Spotify-reported EPs were silently excluded. Spotify lumps EPs and true singles under album_type='single'. The previous _should_include_release short-circuited on album_type='single' and returned the include_singles setting (default False), so 4-6 track EPs on Spotify-primary libraries never survived the filter — even though include_eps defaulted to True. Only 7+ track full albums made it through. This is the main reason users felt the job did nothing. Fixes: - Use the correct deezer_id column name in both reference sites. - Restructure _should_include_release so only 'album', 'ep', and 'compilation' are trusted outright. Anything else (including 'single' and missing type) falls through to a track-count disambiguation matching the download pipeline's _get_album_type_display: 1-3 tracks = true single, 4-6 = EP, 7+ = album. A Spotify-returned 'single' with 5 tracks now correctly counts as an EP. Full suite stays at 263 passed. Ruff clean.
395 lines
15 KiB
Python
395 lines
15 KiB
Python
"""Discography Backfill Job — finds missing albums/tracks for library artists."""
|
|
|
|
from core.metadata_service import (
|
|
get_album_tracks_for_source,
|
|
get_artist_discography,
|
|
get_primary_source,
|
|
MetadataLookupOptions,
|
|
)
|
|
from core.repair_jobs import register_job
|
|
from core.repair_jobs.base import JobContext, JobResult, RepairJob
|
|
from core.watchlist_scanner import (
|
|
is_acoustic_version,
|
|
is_compilation_album,
|
|
is_instrumental_version,
|
|
is_live_version,
|
|
is_remix_version,
|
|
)
|
|
from utils.logging_config import get_logger
|
|
|
|
logger = get_logger("repair_job.discography_backfill")
|
|
|
|
|
|
@register_job
|
|
class DiscographyBackfillJob(RepairJob):
|
|
job_id = 'discography_backfill'
|
|
display_name = 'Discography Backfill'
|
|
description = 'Finds missing albums and tracks for artists in your library'
|
|
help_text = (
|
|
'Scans each artist in your library, fetches their full discography from '
|
|
'the configured metadata source, and adds any tracks you don\'t already '
|
|
'own to the wishlist for automatic download.\n\n'
|
|
'Respects content filters: live versions, remixes, acoustic versions, '
|
|
'instrumentals, and compilations are excluded by default.\n\n'
|
|
'Settings:\n'
|
|
'- Include Albums/EPs/Singles: Which release types to check\n'
|
|
'- Include Live/Remixes/Acoustic/Compilations/Instrumentals: Content type filters\n'
|
|
'- Max Artists Per Run: Limit how many artists to process per scan (default: 50)'
|
|
)
|
|
icon = 'repair-icon-backfill'
|
|
default_enabled = False
|
|
default_interval_hours = 168 # Weekly
|
|
default_settings = {
|
|
'include_albums': True,
|
|
'include_eps': True,
|
|
'include_singles': False,
|
|
'include_live': False,
|
|
'include_remixes': False,
|
|
'include_acoustic': False,
|
|
'include_compilations': False,
|
|
'include_instrumentals': False,
|
|
'max_artists_per_run': 50,
|
|
}
|
|
auto_fix = False
|
|
|
|
def scan(self, context: JobContext) -> JobResult:
|
|
result = JobResult()
|
|
settings = self._get_settings(context)
|
|
|
|
max_artists = settings.get('max_artists_per_run', 50)
|
|
|
|
# Fetch all library artists with their metadata source IDs
|
|
artists = self._get_library_artists(context)
|
|
if not artists:
|
|
logger.info("No artists in library to scan")
|
|
return result
|
|
|
|
total = min(len(artists), max_artists)
|
|
if context.update_progress:
|
|
context.update_progress(0, total)
|
|
if context.report_progress:
|
|
context.report_progress(
|
|
phase=f'Scanning discography for {total} artists...',
|
|
total=total,
|
|
)
|
|
|
|
logger.info("Discography backfill: scanning %d artists (of %d total)", total, len(artists))
|
|
primary_source = get_primary_source()
|
|
|
|
for i, artist in enumerate(artists[:max_artists]):
|
|
if context.check_stop():
|
|
return result
|
|
if i % 5 == 0 and context.wait_if_paused():
|
|
return result
|
|
|
|
artist_id = artist['id']
|
|
artist_name = artist['name']
|
|
|
|
if context.report_progress:
|
|
context.report_progress(
|
|
scanned=i + 1, total=total,
|
|
phase=f'Scanning {i + 1} / {total}',
|
|
log_line=f'Fetching discography: {artist_name}',
|
|
log_type='info',
|
|
)
|
|
|
|
try:
|
|
missing_count = self._scan_artist(context, artist, settings, primary_source, result)
|
|
if missing_count > 0:
|
|
logger.info("Found %d missing tracks for %s", missing_count, artist_name)
|
|
except Exception as e:
|
|
logger.warning("Error scanning discography for %s: %s", artist_name, e)
|
|
result.errors += 1
|
|
|
|
if context.update_progress and (i + 1) % 3 == 0:
|
|
context.update_progress(i + 1, total)
|
|
|
|
# Rate limit between artists
|
|
if context.sleep_or_stop(1.0):
|
|
return result
|
|
|
|
if context.update_progress:
|
|
context.update_progress(total, total)
|
|
|
|
logger.info(
|
|
"Discography backfill complete: %d artists scanned, %d missing tracks found, %d errors",
|
|
result.scanned, result.findings_created, result.errors,
|
|
)
|
|
return result
|
|
|
|
def _scan_artist(self, context, artist, settings, primary_source, result):
|
|
"""Scan one artist's discography and create findings for missing tracks."""
|
|
artist_name = artist['name']
|
|
result.scanned += 1
|
|
|
|
# Build source ID map for more accurate lookups
|
|
source_ids = {}
|
|
if artist.get('spotify_artist_id'):
|
|
source_ids['spotify'] = artist['spotify_artist_id']
|
|
if artist.get('itunes_artist_id'):
|
|
source_ids['itunes'] = artist['itunes_artist_id']
|
|
if artist.get('deezer_id'):
|
|
source_ids['deezer'] = artist['deezer_id']
|
|
|
|
# Fetch full discography
|
|
discography = get_artist_discography(
|
|
artist_id=str(artist['id']),
|
|
artist_name=artist_name,
|
|
options=MetadataLookupOptions(
|
|
allow_fallback=True,
|
|
skip_cache=False,
|
|
artist_source_ids=source_ids if source_ids else None,
|
|
),
|
|
)
|
|
|
|
if not discography:
|
|
result.skipped += 1
|
|
return 0
|
|
|
|
source = discography.get('source', primary_source)
|
|
albums = discography.get('albums', [])
|
|
singles = discography.get('singles', [])
|
|
missing_count = 0
|
|
active_server = None
|
|
if context.config_manager:
|
|
active_server = context.config_manager.get_active_media_server()
|
|
|
|
# Process albums and singles
|
|
for release in albums + singles:
|
|
if context.check_stop():
|
|
return missing_count
|
|
|
|
release_name = release.get('name', '')
|
|
release_id = release.get('id', '')
|
|
total_tracks = release.get('total_tracks', 0) or 0
|
|
album_type = release.get('album_type', 'album')
|
|
release_image = release.get('image_url', '')
|
|
|
|
# Filter by release type
|
|
if not self._should_include_release(total_tracks, album_type, settings):
|
|
continue
|
|
|
|
# Filter compilation albums
|
|
if not settings.get('include_compilations', False):
|
|
if is_compilation_album(release_name):
|
|
continue
|
|
|
|
# Fetch tracks for this release
|
|
try:
|
|
tracks_data = get_album_tracks_for_source(source, str(release_id))
|
|
except Exception:
|
|
tracks_data = None
|
|
|
|
if not tracks_data:
|
|
continue
|
|
|
|
# Extract track items
|
|
items = []
|
|
if isinstance(tracks_data, dict):
|
|
items = tracks_data.get('items', [])
|
|
elif isinstance(tracks_data, list):
|
|
items = tracks_data
|
|
|
|
if not items:
|
|
continue
|
|
|
|
for track_item in items:
|
|
if context.check_stop():
|
|
return missing_count
|
|
|
|
track_name = track_item.get('name', '')
|
|
if not track_name:
|
|
continue
|
|
|
|
# Extract artist name from track
|
|
track_artists = track_item.get('artists', [])
|
|
if track_artists:
|
|
first_artist = track_artists[0]
|
|
if isinstance(first_artist, dict):
|
|
track_artist = first_artist.get('name', artist_name)
|
|
else:
|
|
track_artist = str(first_artist)
|
|
else:
|
|
track_artist = artist_name
|
|
|
|
# Content type filters
|
|
if not settings.get('include_live', False):
|
|
if is_live_version(track_name, release_name):
|
|
continue
|
|
if not settings.get('include_remixes', False):
|
|
if is_remix_version(track_name, release_name):
|
|
continue
|
|
if not settings.get('include_acoustic', False):
|
|
if is_acoustic_version(track_name, release_name):
|
|
continue
|
|
if not settings.get('include_instrumentals', False):
|
|
if is_instrumental_version(track_name, release_name):
|
|
continue
|
|
|
|
# Check if track already exists in library
|
|
db_track, confidence = context.db.check_track_exists(
|
|
track_name, track_artist,
|
|
confidence_threshold=0.7,
|
|
server_source=active_server,
|
|
album=release_name,
|
|
)
|
|
if db_track and confidence >= 0.7:
|
|
continue # Already owned
|
|
|
|
# Check if already in wishlist
|
|
try:
|
|
track_id = track_item.get('id', '')
|
|
if track_id and self._is_in_wishlist(context.db, track_id):
|
|
continue
|
|
except Exception:
|
|
pass
|
|
|
|
# Build track data for wishlist
|
|
track_data = {
|
|
'id': track_item.get('id', f'backfill_{hash(f"{track_artist}_{track_name}") % 100000}'),
|
|
'name': track_name,
|
|
'artists': [{'name': track_artist}],
|
|
'album': {
|
|
'name': release_name,
|
|
'id': str(release_id),
|
|
'images': [{'url': release_image}] if release_image else [],
|
|
'album_type': album_type,
|
|
'release_date': release.get('release_date', ''),
|
|
},
|
|
'duration_ms': track_item.get('duration_ms', 0),
|
|
'track_number': track_item.get('track_number', 0),
|
|
'disc_number': track_item.get('disc_number', 1),
|
|
}
|
|
|
|
# Create finding
|
|
if context.create_finding:
|
|
try:
|
|
context.create_finding(
|
|
job_id=self.job_id,
|
|
finding_type='missing_discography_track',
|
|
severity='info',
|
|
entity_type='track',
|
|
entity_id=str(track_data['id']),
|
|
file_path=None,
|
|
title=f'Missing: {track_name}',
|
|
description=(
|
|
f'"{track_name}" by {track_artist} from '
|
|
f'"{release_name}" is not in your library.'
|
|
),
|
|
details={
|
|
'track_data': track_data,
|
|
'artist_name': artist_name,
|
|
'album_name': release_name,
|
|
'album_image_url': release_image,
|
|
'source': source,
|
|
},
|
|
)
|
|
result.findings_created += 1
|
|
missing_count += 1
|
|
except Exception as e:
|
|
logger.debug("Error creating finding for %s: %s", track_name, e)
|
|
result.errors += 1
|
|
|
|
return missing_count
|
|
|
|
@staticmethod
|
|
def _should_include_release(total_tracks, album_type, settings):
|
|
"""Check if a release should be included based on type settings.
|
|
|
|
Spotify lumps both EPs and true singles under album_type='single', so
|
|
only an explicit 'album' / 'ep' / 'compilation' is trusted outright.
|
|
Anything else (including 'single' or missing type) falls through to a
|
|
track-count disambiguation matching the download pipeline:
|
|
- 1-3 tracks -> true single
|
|
- 4-6 tracks -> EP
|
|
- 7+ tracks -> album
|
|
"""
|
|
normalized = (album_type or '').lower()
|
|
if normalized == 'compilation':
|
|
return settings.get('include_compilations', False)
|
|
if normalized == 'album':
|
|
return settings.get('include_albums', True)
|
|
if normalized == 'ep':
|
|
return settings.get('include_eps', True)
|
|
# 'single' or missing: disambiguate by track count
|
|
if total_tracks >= 7:
|
|
return settings.get('include_albums', True)
|
|
if total_tracks >= 4:
|
|
return settings.get('include_eps', True)
|
|
if total_tracks >= 1:
|
|
return settings.get('include_singles', False)
|
|
return settings.get('include_albums', True)
|
|
|
|
@staticmethod
|
|
def _is_in_wishlist(db, track_id):
|
|
"""Check if a track ID is already in the wishlist."""
|
|
conn = db._get_connection()
|
|
try:
|
|
cursor = conn.cursor()
|
|
cursor.execute(
|
|
"SELECT COUNT(*) FROM wishlist_tracks WHERE spotify_track_id = ?",
|
|
(str(track_id),),
|
|
)
|
|
return cursor.fetchone()[0] > 0
|
|
finally:
|
|
conn.close()
|
|
|
|
def _get_library_artists(self, context):
|
|
"""Get all artists from the library database with source IDs."""
|
|
conn = None
|
|
try:
|
|
conn = context.db._get_connection()
|
|
cursor = conn.cursor()
|
|
|
|
# Check which columns exist
|
|
cursor.execute("PRAGMA table_info(artists)")
|
|
columns = {col[1] for col in cursor.fetchall()}
|
|
|
|
select = ["id", "name"]
|
|
if 'spotify_artist_id' in columns:
|
|
select.append("spotify_artist_id")
|
|
if 'itunes_artist_id' in columns:
|
|
select.append("itunes_artist_id")
|
|
if 'deezer_id' in columns:
|
|
select.append("deezer_id")
|
|
|
|
cursor.execute(f"""
|
|
SELECT {', '.join(select)}
|
|
FROM artists
|
|
WHERE name IS NOT NULL AND name != '' AND name != 'Unknown Artist'
|
|
ORDER BY name
|
|
""")
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
except Exception as e:
|
|
logger.error("Error fetching library artists: %s", e, exc_info=True)
|
|
return []
|
|
finally:
|
|
if conn:
|
|
conn.close()
|
|
|
|
def _get_settings(self, context: JobContext) -> dict:
|
|
if not context.config_manager:
|
|
return self.default_settings.copy()
|
|
cfg = context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {})
|
|
merged = self.default_settings.copy()
|
|
merged.update(cfg)
|
|
return merged
|
|
|
|
def estimate_scope(self, context: JobContext) -> int:
|
|
conn = None
|
|
try:
|
|
conn = context.db._get_connection()
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT COUNT(*) FROM artists
|
|
WHERE name IS NOT NULL AND name != '' AND name != 'Unknown Artist'
|
|
""")
|
|
row = cursor.fetchone()
|
|
settings = self._get_settings(context)
|
|
max_artists = settings.get('max_artists_per_run', 50)
|
|
return min(row[0] if row else 0, max_artists)
|
|
except Exception:
|
|
return 0
|
|
finally:
|
|
if conn:
|
|
conn.close()
|