soulsync/core/repair_jobs/discography_backfill.py

"""Discography Backfill Job — finds missing albums/tracks for library artists."""

from core.metadata_service import (
    get_album_tracks_for_source,
    get_artist_discography,
    get_primary_source,
    MetadataLookupOptions,
)
from core.repair_jobs import register_job
from core.repair_jobs.base import JobContext, JobResult, RepairJob
from core.watchlist_scanner import (
    is_acoustic_version,
    is_compilation_album,
    is_instrumental_version,
    is_live_version,
    is_remix_version,
)
from utils.logging_config import get_logger

logger = get_logger("repair_job.discography_backfill")


@register_job
class DiscographyBackfillJob(RepairJob):
    job_id = 'discography_backfill'
    display_name = 'Discography Backfill'
    description = 'Finds missing albums and tracks for artists in your library'
    help_text = (
        'Scans each artist in your library, fetches their full discography from '
        'the configured metadata source, and creates findings for any tracks '
        'you don\'t already own. Click Fix on a finding to add it to the '
        'wishlist for automatic download.\n\n'
        'Respects content filters: live versions, remixes, acoustic versions, '
        'instrumentals, and compilations are excluded by default.\n\n'
        'Settings:\n'
        '- Max Artists Per Run: Limit how many artists to process per scan (default: 50)\n'
        '- Auto Add To Wishlist: When on, missing tracks are pushed to the wishlist during the scan as well as logged as findings\n'
        '- Include Albums / EPs / Singles: Which release types to check\n'
        '- Include Live / Remixes / Acoustic / Compilations / Instrumentals: Content type filters'
    )
    icon = 'repair-icon-backfill'
    default_enabled = False
    default_interval_hours = 24  # Daily — the scan is rate-limited at 50 artists per run
    # Order matters: the UI renders these in dict-insertion order. Keys beginning
    # with `_section_` are rendered as group headers (not settings rows) and are
    # stripped from the saved config.
    default_settings = {
        '_section_core': 'Core',
        'max_artists_per_run': 50,
        # When on, missing tracks are added to the wishlist during the scan in
        # addition to creating findings. When off (default), only findings are
        # created; the user reviews them and decides per-track in the repair UI.
        'auto_add_to_wishlist': False,
        '_section_release_types': 'Release Types',
        'include_albums': True,
        'include_eps': True,
        'include_singles': True,
        '_section_content_filters': 'Content Filters',
        'include_live': False,
        'include_remixes': False,
        'include_acoustic': False,
        'include_compilations': False,
        'include_instrumentals': False,
    }
    auto_fix = False

    def scan(self, context: JobContext) -> JobResult:
        result = JobResult()
        settings = self._get_settings(context)

        max_artists = settings.get('max_artists_per_run', 50)

        # Fetch all library artists with their metadata source IDs
        artists = self._get_library_artists(context)
        if not artists:
            logger.info("No artists in library to scan")
            return result

        total = min(len(artists), max_artists)
        if context.update_progress:
            context.update_progress(0, total)
        if context.report_progress:
            context.report_progress(
                phase=f'Scanning discography for {total} artists...',
                total=total,
            )

        logger.info("Discography backfill: scanning %d artists (of %d total)", total, len(artists))
        primary_source = get_primary_source()

        for i, artist in enumerate(artists[:max_artists]):
            if context.check_stop():
                return result
            if i % 5 == 0 and context.wait_if_paused():
                return result

            artist_id = artist['id']
            artist_name = artist['name']

            if context.report_progress:
                context.report_progress(
                    scanned=i + 1, total=total,
                    phase=f'Scanning {i + 1} / {total}',
                    log_line=f'Fetching discography: {artist_name}',
                    log_type='info',
                )

            logger.info("[%d/%d] Scanning %s", i + 1, total, artist_name)
            try:
                missing_count = self._scan_artist(context, artist, settings, primary_source, result)
                if missing_count > 0:
                    logger.info("[%d/%d] Found %d missing tracks for %s", i + 1, total, missing_count, artist_name)
                else:
                    logger.info("[%d/%d] %s — no missing tracks", i + 1, total, artist_name)
            except Exception as e:
                logger.warning("[%d/%d] Error scanning discography for %s: %s", i + 1, total, artist_name, e)
                result.errors += 1

            if context.update_progress and (i + 1) % 3 == 0:
                context.update_progress(i + 1, total)

            # Rate limit between artists
            if context.sleep_or_stop(1.0):
                return result

        if context.update_progress:
            context.update_progress(total, total)

        logger.info(
            "Discography backfill complete: %d artists scanned, %d missing tracks found, %d errors",
            result.scanned, result.findings_created, result.errors,
        )
        return result

    def _scan_artist(self, context, artist, settings, primary_source, result):
        """Scan one artist's discography and create findings for missing tracks.

        Uses the same batched in-memory matching the Library and Artists pages
        use (get_candidate_albums_for_artist + get_candidate_tracks_for_albums)
        so one artist with a big library doesn't trigger thousands of per-track
        SQL queries.
        """
        artist_name = artist['name']
        result.scanned += 1

        # Build source ID map for more accurate lookups. Primary fallback
        # relies on artist-name search when a source ID is missing.
        source_ids = {}
        if artist.get('spotify_artist_id'):
            source_ids['spotify'] = artist['spotify_artist_id']
        if artist.get('itunes_artist_id'):
            source_ids['itunes'] = artist['itunes_artist_id']
        if artist.get('deezer_id'):
            source_ids['deezer'] = artist['deezer_id']

        # Fetch full discography
        discography = get_artist_discography(
            artist_id=str(artist['id']),
            artist_name=artist_name,
            options=MetadataLookupOptions(
                allow_fallback=True,
                skip_cache=False,
                artist_source_ids=source_ids if source_ids else None,
            ),
        )

        if not discography:
            result.skipped += 1
            return 0

        source = discography.get('source', primary_source)
        albums = discography.get('albums', [])
        singles = discography.get('singles', [])
        missing_count = 0
        active_server = None
        if context.config_manager:
            active_server = context.config_manager.get_active_media_server()

        auto_add = settings.get('auto_add_to_wishlist', False)

        # Pre-fetch the artist's library albums + tracks ONCE per artist for
        # fast in-memory matching (same pattern as the Library/Artists page
        # completion check). Avoids thousands of per-track SQL calls.
        candidate_tracks = None
        try:
            cand_albums = context.db.get_candidate_albums_for_artist(
                artist_name, server_source=active_server
            )
            if cand_albums:
                candidate_tracks = context.db.get_candidate_tracks_for_albums(
                    [a.id for a in cand_albums]
                )
        except Exception as exc:
            logger.debug("Could not pre-fetch candidates for %s: %s", artist_name, exc)
            candidate_tracks = None

        # Process albums and singles
        for release in albums + singles:
            if context.check_stop():
                return missing_count

            release_name = release.get('name', '')
            release_id = release.get('id', '')
            total_tracks = release.get('total_tracks', 0) or 0
            album_type = release.get('album_type', 'album')
            release_image = release.get('image_url', '') or ''
            release_date = release.get('release_date', '') or ''

            # Filter by release type
            if not self._should_include_release(total_tracks, album_type, settings):
                continue

            # Filter compilation albums
            if not settings.get('include_compilations', False):
                if is_compilation_album(release_name):
                    continue

            # Fetch tracks for this release
            try:
                tracks_data = get_album_tracks_for_source(source, str(release_id))
            except Exception:
                tracks_data = None

            if not tracks_data:
                continue

            # Extract track items
            items = []
            if isinstance(tracks_data, dict):
                items = tracks_data.get('items', [])
            elif isinstance(tracks_data, list):
                items = tracks_data

            if not items:
                continue

            # Build the full album context once per release so every finding
            # created for this release carries the same wishlist-ready dict.
            # Matches the shape add_to_wishlist / download pipeline expects.
            album_context = {
                'id': str(release_id),
                'name': release_name,
                'album_type': album_type,
                'release_date': release_date,
                'images': [{'url': release_image}] if release_image else [],
                'image_url': release_image,
                'artists': [{'name': artist_name}],
                'total_tracks': total_tracks,
            }

            for track_item in items:
                if context.check_stop():
                    return missing_count

                track_name = track_item.get('name', '')
                if not track_name:
                    continue

                # Extract artist name from track (fall back to the discography artist)
                track_artists = track_item.get('artists', [])
                if track_artists:
                    first_artist = track_artists[0]
                    if isinstance(first_artist, dict):
                        track_artist = first_artist.get('name', artist_name)
                    else:
                        track_artist = str(first_artist)
                else:
                    track_artist = artist_name

                # Content type filters
                if not settings.get('include_live', False):
                    if is_live_version(track_name, release_name):
                        continue
                if not settings.get('include_remixes', False):
                    if is_remix_version(track_name, release_name):
                        continue
                if not settings.get('include_acoustic', False):
                    if is_acoustic_version(track_name, release_name):
                        continue
                if not settings.get('include_instrumentals', False):
                    if is_instrumental_version(track_name, release_name):
                        continue

                # Check if track already exists in library — batched in-memory
                # match when candidates were pre-fetched (fast path). Falls back
                # to the legacy SQL path if pre-fetch failed.
                db_track, confidence = context.db.check_track_exists(
                    track_name, track_artist,
                    confidence_threshold=0.7,
                    server_source=active_server,
                    album=release_name,
                    candidate_tracks=candidate_tracks,
                )
                if db_track and confidence >= 0.7:
                    continue  # Already owned

                # Check if already in wishlist
                try:
                    track_id = track_item.get('id', '')
                    if track_id and self._is_in_wishlist(context.db, track_id):
                        continue
                except Exception as e:
                    logger.debug("wishlist membership check failed: %s", e)

                # Build wishlist-ready track data. album is a dict (required by
                # add_to_wishlist and by the download pipeline's cover-art
                # extraction). Every finding carries enough context that the
                # fix handler can hand it straight to the wishlist.
                track_data = {
                    'id': track_item.get('id', f'backfill_{hash(f"{track_artist}_{track_name}") % 100000}'),
                    'name': track_name,
                    'artists': [{'name': track_artist}],
                    'album': dict(album_context),  # copy so per-track mutations don't bleed
                    'duration_ms': track_item.get('duration_ms', 0),
                    'track_number': track_item.get('track_number', 0),
                    'disc_number': track_item.get('disc_number', 1),
                    'image_url': release_image,
                }

                # Create finding
                if context.create_finding:
                    try:
                        inserted = context.create_finding(
                            job_id=self.job_id,
                            finding_type='missing_discography_track',
                            severity='info',
                            entity_type='track',
                            entity_id=str(track_data['id']),
                            file_path=None,
                            title=f'Missing: {track_name}',
                            description=(
                                f'"{track_name}" by {track_artist} from '
                                f'"{release_name}" is not in your library.'
                            ),
                            details={
                                'track_data': track_data,
                                'artist_name': artist_name,
                                'album_name': release_name,
                                'album_image_url': release_image,
                                'source': source,
                            },
                        )
                        if inserted:
                            result.findings_created += 1
                            missing_count += 1
                        else:
                            result.findings_skipped_dedup += 1

                        # Auto-wishlist mode: also push to wishlist now. The
                        # finding still gets created so the user has a log of
                        # what the backfill picked up. Only fire on a NEW
                        # finding — skip if dedup-suppressed (already on the
                        # wishlist or already auto-added in a prior scan).
                        if auto_add and inserted:
                            try:
                                context.db.add_to_wishlist(
                                    spotify_track_data=track_data,
                                    failure_reason='Discography backfill — missing from library (auto-added)',
                                    source_type='repair',
                                    source_info={
                                        'job': 'discography_backfill',
                                        'artist': artist_name,
                                        'auto_added': True,
                                    },
                                )
                            except Exception as wl_err:
                                logger.debug("Auto-add to wishlist failed for '%s': %s", track_name, wl_err)
                    except Exception as e:
                        logger.debug("Error creating finding for %s: %s", track_name, e)
                        result.errors += 1

        return missing_count

    @staticmethod
    def _should_include_release(total_tracks, album_type, settings):
        """Check if a release should be included based on type settings.

        Spotify lumps both EPs and true singles under album_type='single', so
        only an explicit 'album' / 'ep' / 'compilation' is trusted outright.
        Anything else (including 'single' or missing type) falls through to a
        track-count disambiguation matching the download pipeline:
          - 1-3 tracks -> true single
          - 4-6 tracks -> EP
          - 7+ tracks -> album
        """
        normalized = (album_type or '').lower()
        if normalized == 'compilation':
            return settings.get('include_compilations', False)
        if normalized == 'album':
            return settings.get('include_albums', True)
        if normalized == 'ep':
            return settings.get('include_eps', True)
        # 'single' or missing: disambiguate by track count
        if total_tracks >= 7:
            return settings.get('include_albums', True)
        if total_tracks >= 4:
            return settings.get('include_eps', True)
        if total_tracks >= 1:
            return settings.get('include_singles', False)
        return settings.get('include_albums', True)

    @staticmethod
    def _is_in_wishlist(db, track_id):
        """Check if a track ID is already in the wishlist."""
        conn = db._get_connection()
        try:
            cursor = conn.cursor()
            cursor.execute(
                "SELECT COUNT(*) FROM wishlist_tracks WHERE spotify_track_id = ?",
                (str(track_id),),
            )
            return cursor.fetchone()[0] > 0
        finally:
            conn.close()

    def _get_library_artists(self, context):
        """Get all artists from the library database with source IDs."""
        conn = None
        try:
            conn = context.db._get_connection()
            cursor = conn.cursor()

            # Check which columns exist
            cursor.execute("PRAGMA table_info(artists)")
            columns = {col[1] for col in cursor.fetchall()}

            select = ["id", "name"]
            if 'spotify_artist_id' in columns:
                select.append("spotify_artist_id")
            if 'itunes_artist_id' in columns:
                select.append("itunes_artist_id")
            if 'deezer_id' in columns:
                select.append("deezer_id")

            cursor.execute(f"""
                SELECT {', '.join(select)}
                FROM artists
                WHERE name IS NOT NULL AND name != '' AND name != 'Unknown Artist'
                ORDER BY name
            """)
            return [dict(row) for row in cursor.fetchall()]
        except Exception as e:
            logger.error("Error fetching library artists: %s", e, exc_info=True)
            return []
        finally:
            if conn:
                conn.close()

    def _get_settings(self, context: JobContext) -> dict:
        if not context.config_manager:
            return self.default_settings.copy()
        cfg = context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {})
        merged = self.default_settings.copy()
        merged.update(cfg)
        return merged

    def estimate_scope(self, context: JobContext) -> int:
        conn = None
        try:
            conn = context.db._get_connection()
            cursor = conn.cursor()
            cursor.execute("""
                SELECT COUNT(*) FROM artists
                WHERE name IS NOT NULL AND name != '' AND name != 'Unknown Artist'
            """)
            row = cursor.fetchone()
            settings = self._get_settings(context)
            max_artists = settings.get('max_artists_per_run', 50)
            return min(row[0] if row else 0, max_artists)
        except Exception:
            return 0
        finally:
            if conn:
                conn.close()