diff --git a/core/repair_jobs/__init__.py b/core/repair_jobs/__init__.py index 159dc7bc..c6f780e8 100644 --- a/core/repair_jobs/__init__.py +++ b/core/repair_jobs/__init__.py @@ -43,6 +43,7 @@ _JOB_MODULES = [ 'core.repair_jobs.album_tag_consistency', 'core.repair_jobs.live_commentary_cleaner', 'core.repair_jobs.unknown_artist_fixer', + 'core.repair_jobs.discography_backfill', ] diff --git a/core/repair_jobs/discography_backfill.py b/core/repair_jobs/discography_backfill.py new file mode 100644 index 00000000..6b307293 --- /dev/null +++ b/core/repair_jobs/discography_backfill.py @@ -0,0 +1,387 @@ +"""Discography Backfill Job — finds missing albums/tracks for library artists.""" + +from core.metadata_service import ( + get_album_tracks_for_source, + get_artist_discography, + get_primary_source, + MetadataLookupOptions, +) +from core.repair_jobs import register_job +from core.repair_jobs.base import JobContext, JobResult, RepairJob +from core.watchlist_scanner import ( + is_acoustic_version, + is_compilation_album, + is_instrumental_version, + is_live_version, + is_remix_version, +) +from utils.logging_config import get_logger + +logger = get_logger("repair_job.discography_backfill") + + +@register_job +class DiscographyBackfillJob(RepairJob): + job_id = 'discography_backfill' + display_name = 'Discography Backfill' + description = 'Finds missing albums and tracks for artists in your library' + help_text = ( + 'Scans each artist in your library, fetches their full discography from ' + 'the configured metadata source, and adds any tracks you don\'t already ' + 'own to the wishlist for automatic download.\n\n' + 'Respects content filters: live versions, remixes, acoustic versions, ' + 'instrumentals, and compilations are excluded by default.\n\n' + 'Settings:\n' + '- Include Albums/EPs/Singles: Which release types to check\n' + '- Include Live/Remixes/Acoustic/Compilations/Instrumentals: Content type filters\n' + '- Max Artists Per Run: Limit how many artists to process per scan (default: 50)' + ) + icon = 'repair-icon-backfill' + default_enabled = False + default_interval_hours = 168 # Weekly + default_settings = { + 'include_albums': True, + 'include_eps': True, + 'include_singles': False, + 'include_live': False, + 'include_remixes': False, + 'include_acoustic': False, + 'include_compilations': False, + 'include_instrumentals': False, + 'max_artists_per_run': 50, + } + auto_fix = False + + def scan(self, context: JobContext) -> JobResult: + result = JobResult() + settings = self._get_settings(context) + + max_artists = settings.get('max_artists_per_run', 50) + + # Fetch all library artists with their metadata source IDs + artists = self._get_library_artists(context) + if not artists: + logger.info("No artists in library to scan") + return result + + total = min(len(artists), max_artists) + if context.update_progress: + context.update_progress(0, total) + if context.report_progress: + context.report_progress( + phase=f'Scanning discography for {total} artists...', + total=total, + ) + + logger.info("Discography backfill: scanning %d artists (of %d total)", total, len(artists)) + primary_source = get_primary_source() + + for i, artist in enumerate(artists[:max_artists]): + if context.check_stop(): + return result + if i % 5 == 0 and context.wait_if_paused(): + return result + + artist_id = artist['id'] + artist_name = artist['name'] + + if context.report_progress: + context.report_progress( + scanned=i + 1, total=total, + phase=f'Scanning {i + 1} / {total}', + log_line=f'Fetching discography: {artist_name}', + log_type='info', + ) + + try: + missing_count = self._scan_artist(context, artist, settings, primary_source, result) + if missing_count > 0: + logger.info("Found %d missing tracks for %s", missing_count, artist_name) + except Exception as e: + logger.warning("Error scanning discography for %s: %s", artist_name, e) + result.errors += 1 + + if context.update_progress and (i + 1) % 3 == 0: + context.update_progress(i + 1, total) + + # Rate limit between artists + if context.sleep_or_stop(1.0): + return result + + if context.update_progress: + context.update_progress(total, total) + + logger.info( + "Discography backfill complete: %d artists scanned, %d missing tracks found, %d errors", + result.scanned, result.findings_created, result.errors, + ) + return result + + def _scan_artist(self, context, artist, settings, primary_source, result): + """Scan one artist's discography and create findings for missing tracks.""" + artist_name = artist['name'] + result.scanned += 1 + + # Build source ID map for more accurate lookups + source_ids = {} + if artist.get('spotify_artist_id'): + source_ids['spotify'] = artist['spotify_artist_id'] + if artist.get('itunes_artist_id'): + source_ids['itunes'] = artist['itunes_artist_id'] + if artist.get('deezer_artist_id'): + source_ids['deezer'] = artist['deezer_artist_id'] + + # Fetch full discography + discography = get_artist_discography( + artist_id=str(artist['id']), + artist_name=artist_name, + options=MetadataLookupOptions( + allow_fallback=True, + skip_cache=False, + artist_source_ids=source_ids if source_ids else None, + ), + ) + + if not discography: + result.skipped += 1 + return 0 + + source = discography.get('source', primary_source) + albums = discography.get('albums', []) + singles = discography.get('singles', []) + missing_count = 0 + active_server = None + if context.config_manager: + active_server = context.config_manager.get_active_media_server() + + # Process albums and singles + for release in albums + singles: + if context.check_stop(): + return missing_count + + release_name = release.get('name', '') + release_id = release.get('id', '') + total_tracks = release.get('total_tracks', 0) or 0 + album_type = release.get('album_type', 'album') + release_image = release.get('image_url', '') + + # Filter by release type + if not self._should_include_release(total_tracks, album_type, settings): + continue + + # Filter compilation albums + if not settings.get('include_compilations', False): + if is_compilation_album(release_name): + continue + + # Fetch tracks for this release + try: + tracks_data = get_album_tracks_for_source(source, str(release_id)) + except Exception: + tracks_data = None + + if not tracks_data: + continue + + # Extract track items + items = [] + if isinstance(tracks_data, dict): + items = tracks_data.get('items', []) + elif isinstance(tracks_data, list): + items = tracks_data + + if not items: + continue + + for track_item in items: + if context.check_stop(): + return missing_count + + track_name = track_item.get('name', '') + if not track_name: + continue + + # Extract artist name from track + track_artists = track_item.get('artists', []) + if track_artists: + first_artist = track_artists[0] + if isinstance(first_artist, dict): + track_artist = first_artist.get('name', artist_name) + else: + track_artist = str(first_artist) + else: + track_artist = artist_name + + # Content type filters + if not settings.get('include_live', False): + if is_live_version(track_name, release_name): + continue + if not settings.get('include_remixes', False): + if is_remix_version(track_name, release_name): + continue + if not settings.get('include_acoustic', False): + if is_acoustic_version(track_name, release_name): + continue + if not settings.get('include_instrumentals', False): + if is_instrumental_version(track_name, release_name): + continue + + # Check if track already exists in library + db_track, confidence = context.db.check_track_exists( + track_name, track_artist, + confidence_threshold=0.7, + server_source=active_server, + album=release_name, + ) + if db_track and confidence >= 0.7: + continue # Already owned + + # Check if already in wishlist + try: + track_id = track_item.get('id', '') + if track_id and self._is_in_wishlist(context.db, track_id): + continue + except Exception: + pass + + # Build track data for wishlist + track_data = { + 'id': track_item.get('id', f'backfill_{hash(f"{track_artist}_{track_name}") % 100000}'), + 'name': track_name, + 'artists': [{'name': track_artist}], + 'album': { + 'name': release_name, + 'id': str(release_id), + 'images': [{'url': release_image}] if release_image else [], + 'album_type': album_type, + 'release_date': release.get('release_date', ''), + }, + 'duration_ms': track_item.get('duration_ms', 0), + 'track_number': track_item.get('track_number', 0), + 'disc_number': track_item.get('disc_number', 1), + } + + # Create finding + if context.create_finding: + try: + context.create_finding( + job_id=self.job_id, + finding_type='missing_discography_track', + severity='info', + entity_type='track', + entity_id=str(track_data['id']), + file_path=None, + title=f'Missing: {track_name}', + description=( + f'"{track_name}" by {track_artist} from ' + f'"{release_name}" is not in your library.' + ), + details={ + 'track_data': track_data, + 'artist_name': artist_name, + 'album_name': release_name, + 'album_image_url': release_image, + 'source': source, + }, + ) + result.findings_created += 1 + missing_count += 1 + except Exception as e: + logger.debug("Error creating finding for %s: %s", track_name, e) + result.errors += 1 + + return missing_count + + @staticmethod + def _should_include_release(total_tracks, album_type, settings): + """Check if a release should be included based on type settings.""" + # Use album_type from metadata source when available + normalized = (album_type or '').lower() + if normalized == 'compilation': + return settings.get('include_compilations', False) + if normalized in ('single',): + return settings.get('include_singles', False) + if normalized in ('ep',): + return settings.get('include_eps', True) + # Fall back to track count heuristic + if total_tracks >= 7: + return settings.get('include_albums', True) + elif total_tracks >= 4: + return settings.get('include_eps', True) + elif total_tracks >= 1: + return settings.get('include_singles', False) + return settings.get('include_albums', True) + + @staticmethod + def _is_in_wishlist(db, track_id): + """Check if a track ID is already in the wishlist.""" + conn = db._get_connection() + try: + cursor = conn.cursor() + cursor.execute( + "SELECT COUNT(*) FROM wishlist_tracks WHERE spotify_track_id = ?", + (str(track_id),), + ) + return cursor.fetchone()[0] > 0 + finally: + conn.close() + + def _get_library_artists(self, context): + """Get all artists from the library database with source IDs.""" + conn = None + try: + conn = context.db._get_connection() + cursor = conn.cursor() + + # Check which columns exist + cursor.execute("PRAGMA table_info(artists)") + columns = {col[1] for col in cursor.fetchall()} + + select = ["id", "name"] + if 'spotify_artist_id' in columns: + select.append("spotify_artist_id") + if 'itunes_artist_id' in columns: + select.append("itunes_artist_id") + if 'deezer_artist_id' in columns: + select.append("deezer_artist_id") + + cursor.execute(f""" + SELECT {', '.join(select)} + FROM artists + WHERE name IS NOT NULL AND name != '' AND name != 'Unknown Artist' + ORDER BY name + """) + return [dict(row) for row in cursor.fetchall()] + except Exception as e: + logger.error("Error fetching library artists: %s", e, exc_info=True) + return [] + finally: + if conn: + conn.close() + + def _get_settings(self, context: JobContext) -> dict: + if not context.config_manager: + return self.default_settings.copy() + cfg = context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {}) + merged = self.default_settings.copy() + merged.update(cfg) + return merged + + def estimate_scope(self, context: JobContext) -> int: + conn = None + try: + conn = context.db._get_connection() + cursor = conn.cursor() + cursor.execute(""" + SELECT COUNT(*) FROM artists + WHERE name IS NOT NULL AND name != '' AND name != 'Unknown Artist' + """) + row = cursor.fetchone() + settings = self._get_settings(context) + max_artists = settings.get('max_artists_per_run', 50) + return min(row[0] if row else 0, max_artists) + except Exception: + return 0 + finally: + if conn: + conn.close() diff --git a/core/repair_worker.py b/core/repair_worker.py index 207f9df0..0810677f 100644 --- a/core/repair_worker.py +++ b/core/repair_worker.py @@ -845,12 +845,33 @@ class RepairWorker: 'unwanted_content': self._fix_unwanted_content, 'unknown_artist': self._fix_unknown_artist, 'acoustid_mismatch': self._fix_acoustid_mismatch, + 'missing_discography_track': self._fix_discography_backfill, } handler = handlers.get(finding_type) if not handler: return {'success': False, 'error': f'No fix available for finding type: {finding_type}'} return handler(entity_type, entity_id, file_path, details) + def _fix_discography_backfill(self, entity_type, entity_id, file_path, details): + """Add missing discography track to wishlist.""" + track_data = details.get('track_data') + if not track_data: + return {'success': False, 'error': 'No track data in finding'} + try: + success = self.db.add_to_wishlist( + spotify_track_data=track_data, + failure_reason='Discography backfill — missing from library', + source_type='repair', + source_info={'job': 'discography_backfill', 'artist': details.get('artist_name', '')} + ) + track_name = track_data.get('name', '?') + if success: + return {'success': True, 'action': 'added_to_wishlist', + 'message': f"Added '{track_name}' to wishlist"} + return {'success': False, 'error': f"Could not add '{track_name}' to wishlist (may already exist)"} + except Exception as e: + return {'success': False, 'error': str(e)} + def _fix_dead_file(self, entity_type, entity_id, file_path, details): """Fix a dead file reference. Action depends on details['_fix_action']: 'redownload' (default) — add to wishlist + remove DB entry diff --git a/webui/static/script.js b/webui/static/script.js index e9a3aacc..96b406f1 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -65544,6 +65544,7 @@ async function loadRepairFindings() { incomplete_album: 'Auto-Fill', missing_lossy_copy: 'Convert', acoustid_mismatch: 'Fix', + missing_discography_track: 'Add to Wishlist', }; container.innerHTML = items.map(f => {