import os
import json
import asyncio
import requests
import socket
import ipaddress
import subprocess
import platform
import threading
import time
import shutil
import glob
import uuid
import re
import sqlite3
from pathlib import Path
from urllib.parse import urljoin

from concurrent.futures import ThreadPoolExecutor, as_completed
from flask import Flask, render_template, request, jsonify, redirect, send_file, Response, session, g
from flask_socketio import SocketIO, emit, join_room, leave_room
from utils.logging_config import get_logger
from utils.async_helpers import run_async

# --- Core Application Imports ---
# Import the same core clients and config manager used by the GUI app
from config.settings import config_manager

# Initialize logger
logger = get_logger("web_server")

# Dedicated source reuse logger — writes to logs/source_reuse.log
import logging as _logging
import logging.handlers as _logging_handlers
source_reuse_logger = _logging.getLogger("source_reuse")
source_reuse_logger.setLevel(_logging.DEBUG)
if not source_reuse_logger.handlers:
    _sr_handler = _logging_handlers.RotatingFileHandler(
        "logs/source_reuse.log", encoding="utf-8", maxBytes=5*1024*1024, backupCount=2
    )
    _sr_handler.setFormatter(_logging.Formatter("%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"))
    source_reuse_logger.addHandler(_sr_handler)
    source_reuse_logger.propagate = False

# Dedicated post-processing logger (failures only) — writes to logs/post_processing.log
pp_logger = _logging.getLogger("post_processing")
pp_logger.setLevel(_logging.DEBUG)
if not pp_logger.handlers:
    _pp_handler = _logging_handlers.RotatingFileHandler(
        "logs/post_processing.log", encoding="utf-8", maxBytes=5*1024*1024, backupCount=2
    )
    _pp_handler.setFormatter(_logging.Formatter("%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"))
    pp_logger.addHandler(_pp_handler)
    pp_logger.propagate = False
from core.spotify_client import SpotifyClient, Playlist as SpotifyPlaylist, Track as SpotifyTrack
from core.plex_client import PlexClient
from core.jellyfin_client import JellyfinClient
from core.navidrome_client import NavidromeClient
from core.soulseek_client import SoulseekClient
from core.download_orchestrator import DownloadOrchestrator
from core.tidal_client import TidalClient # Added import for Tidal
from core.matching_engine import MusicMatchingEngine
from core.database_update_worker import DatabaseUpdateWorker, DatabaseStatsWorker
from core.web_scan_manager import WebScanManager
from core.lyrics_client import lyrics_client
from database.music_database import get_database
from services.sync_service import PlaylistSyncService
from datetime import datetime
import yt_dlp
from core.matching_engine import MusicMatchingEngine
from beatport_unified_scraper import BeatportUnifiedScraper
from core.musicbrainz_worker import MusicBrainzWorker
from core.audiodb_worker import AudioDBWorker
from core.deezer_worker import DeezerWorker
from core.spotify_worker import SpotifyWorker
from core.itunes_worker import iTunesWorker
from core.hydrabase_worker import HydrabaseWorker
from core.hydrabase_client import HydrabaseClient
from core.automation_engine import AutomationEngine

# --- Flask App Setup ---
base_dir = os.path.abspath(os.path.dirname(__file__))
project_root = os.path.dirname(base_dir) # Go up one level to the project root

# Check for environment variable first (Docker support), then fallback to calculated path
env_config_path = os.environ.get('SOULSYNC_CONFIG_PATH')
if env_config_path:
    config_path = env_config_path
    print(f"🔧 Using config path from environment: {config_path}")
else:
    config_path = os.path.join(project_root, 'config', 'config.json')

if os.path.exists(config_path):
    # Check if we need to reload or if settings.py already handled it
    current_loaded_path = getattr(config_manager, 'config_path', None)
    target_path = Path(config_path).resolve()

    # Resolve current loaded path if it's a Path object
    if isinstance(current_loaded_path, Path):
        current_loaded_path = current_loaded_path.resolve()

    if current_loaded_path == target_path and config_manager.config_data:
        print(f"✅ Web server configuration already loaded from: {config_path}")
    else:
        print(f"Found config file at: {config_path}")
        # Load configuration into the existing singleton instance
        if hasattr(config_manager, 'load_config'):
            config_manager.load_config(config_path)
        else:
            # Fallback for older settings.py in Docker volumes
            print("⚠️ Legacy configuration detected: using fallback loading method")
            config_manager.config_path = Path(config_path)
            config_manager._load_config()
        print("✅ Web server configuration loaded successfully.")
else:
    print(f"🔴 WARNING: config.json not found at {config_path}. Using default settings.")
# Correctly point to the 'webui' directory for templates and static files
app = Flask(
    __name__,
    template_folder=os.path.join(base_dir, 'webui'),
    static_folder=os.path.join(base_dir, 'webui', 'static')
)

# --- Flask Session Setup (for multi-profile support) ---
import secrets as _secrets
def _init_flask_secret_key():
    """Load or generate a persistent secret key for Flask sessions"""
    try:
        db = get_database()
        key = db.get_metadata('flask_secret_key')
        if not key:
            key = _secrets.token_hex(32)
            db.set_metadata('flask_secret_key', key)
        return key
    except Exception:
        return _secrets.token_hex(32)

app.secret_key = _init_flask_secret_key()

# --- WebSocket (Socket.IO) Setup ---
socketio = SocketIO(app, async_mode='threading', cors_allowed_origins='*')

# --- Profile Context (before_request hook) ---
@app.before_request
def _set_profile_context():
    """Set g.profile_id from session for every request"""
    # Skip for profile management, static, and root routes
    path = request.path
    if (path.startswith('/api/profiles') or
        path.startswith('/static/') or
        path == '/' or
        path.startswith('/api/v1/')):
        g.profile_id = session.get('profile_id', 1)
        return

    pid = session.get('profile_id', 1)

    # Validate session profile still exists (handles deleted profiles)
    if pid != 1 and 'profile_id' in session:
        try:
            database = get_database()
            profile = database.get_profile(pid)
            if not profile:
                session.pop('profile_id', None)
                from flask import jsonify as _jsonify
                return _jsonify({"error": "profile_required", "message": "Profile no longer exists"}), 401
        except Exception:
            pass  # DB error — don't block requests, use the session value

    g.profile_id = pid

def get_current_profile_id() -> int:
    """Get the current profile ID from Flask g context or default to 1"""
    try:
        return g.profile_id
    except AttributeError:
        return 1

# --- Docker Helper Functions ---
def docker_resolve_path(path_str):
    """
    Resolve absolute paths for Docker container access
    In Docker, Windows drive paths (E:/) need to be mapped to WSL mount points (/mnt/e/)
    """
    if os.path.exists('/.dockerenv') and len(path_str) >= 3 and path_str[1] == ':' and path_str[0].isalpha():
        # Convert Windows path (E:/path) to WSL mount path (/mnt/e/path)
        drive_letter = path_str[0].lower()
        rest_of_path = path_str[2:].replace('\\', '/')  # Remove E: and convert backslashes
        return f"/host/mnt/{drive_letter}{rest_of_path}"
    return path_str

def extract_filename(full_path):
    """
    Extract filename by working backwards from the end until we hit a separator.
    This is cross-platform compatible and handles both Windows and Unix path separators.

    Special handling for YouTube/Tidal: If the filename contains '||' (encoded format),
    treat it as a filename, not a path, to avoid splitting on '/' in titles.
    """
    if not full_path:
        return ""

    # YouTube filenames are encoded as "video_id||title" and may contain '/' in the title
    # Don't split these on path separators
    if '||' in full_path:
        return full_path

    last_slash = max(full_path.rfind('/'), full_path.rfind('\\'))
    if last_slash != -1:
        return full_path[last_slash + 1:]
    else:
        return full_path

def _make_context_key(username, filename):
    """Build a unique context key from username and full Soulseek path.

    Uses the full remote path (not just filename) to prevent collisions
    when different tracks from the same user share a filename
    (e.g., two albums both containing '01 - Intro.flac').
    """
    normalized = filename.replace('\\', '/').lstrip('/') if filename else ''
    return f"{username}::{normalized}"

# --- Initialize Core Application Components ---
print("🚀 Initializing SoulSync services for Web UI...")
try:
    spotify_client = SpotifyClient()
    plex_client = PlexClient()
    jellyfin_client = JellyfinClient()
    navidrome_client = NavidromeClient()
    # Use DownloadOrchestrator instead of SoulseekClient directly (routes between Soulseek/YouTube)
    soulseek_client = DownloadOrchestrator()
    tidal_client = TidalClient()
    matching_engine = MusicMatchingEngine()
    sync_service = PlaylistSyncService(spotify_client, plex_client, soulseek_client, jellyfin_client, navidrome_client)
    
    # Inject shutdown check callback into YouTube and Tidal clients (avoids circular imports)
    # The callback uses the global IS_SHUTTING_DOWN flag from this module
    if hasattr(soulseek_client, 'youtube'):
         soulseek_client.youtube.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
         print("✅ Configured YouTube client shutdown callback")
    if hasattr(soulseek_client, 'tidal'):
         soulseek_client.tidal.set_shutdown_check(lambda: IS_SHUTTING_DOWN)
         print("✅ Configured Tidal download client shutdown callback")

    # Initialize web scan manager for automatic post-download scanning
    media_clients = {
        'plex_client': plex_client,
        'jellyfin_client': jellyfin_client,
        'navidrome_client': navidrome_client
    }
    web_scan_manager = WebScanManager(media_clients, delay_seconds=60)
    print("✅ Core service clients and scan manager initialized.")
except Exception as e:
    print(f"🔴 FATAL: Error initializing service clients: {e}")
    spotify_client = plex_client = jellyfin_client = navidrome_client = soulseek_client = tidal_client = matching_engine = sync_service = web_scan_manager = None

# --- Automation Engine ---
try:
    automation_engine = AutomationEngine(get_database())
    print("✅ Automation engine initialized.")
except Exception as e:
    print(f"⚠️ Automation engine failed to initialize: {e}")
    automation_engine = None

def _register_automation_handlers():
    """Register real SoulSync action handlers with the automation engine."""
    if not automation_engine:
        return

    def _auto_process_wishlist(config):
        try:
            _process_wishlist_automatically(automation_id=config.get('_automation_id'))
            return {'status': 'completed'}
        except Exception as e:
            return {'status': 'error', 'error': str(e)}

    def _auto_scan_watchlist(config):
        try:
            _process_watchlist_scan_automatically(automation_id=config.get('_automation_id'))
            return {'status': 'completed'}
        except Exception as e:
            return {'status': 'error', 'error': str(e)}

    def _auto_scan_library(config):
        if web_scan_manager:
            result = web_scan_manager.request_scan('Automation trigger')
            return {'status': result.get('status', 'unknown')}
        return {'status': 'error', 'reason': 'Scan manager not available'}

    cross_guard = lambda: is_wishlist_actually_processing() or is_watchlist_actually_scanning()
    automation_engine.register_action_handler('process_wishlist', _auto_process_wishlist, cross_guard)
    automation_engine.register_action_handler('scan_watchlist', _auto_scan_watchlist, cross_guard)
    automation_engine.register_action_handler('scan_library', _auto_scan_library)

    def _auto_refresh_mirrored(config):
        """Refresh mirrored playlist(s) from source."""
        db = get_database()
        playlist_id = config.get('playlist_id')
        refresh_all = config.get('all', False)
        auto_id = config.get('_automation_id')

        if refresh_all:
            playlists = db.get_mirrored_playlists()
        elif playlist_id:
            p = db.get_mirrored_playlist(int(playlist_id))
            playlists = [p] if p else []
        else:
            return {'status': 'error', 'reason': 'No playlist specified'}

        refreshed = 0
        errors = []
        for idx, pl in enumerate(playlists):
            try:
                source = pl.get('source', '')
                source_id = pl.get('source_playlist_id', '')
                _update_automation_progress(auto_id,
                    progress=(idx / max(1, len(playlists))) * 100,
                    phase=f'Refreshing: "{pl.get("name", "")}"',
                    current_item=pl.get('name', ''))
                tracks = None

                if source == 'spotify' and spotify_client and spotify_client.is_spotify_authenticated():
                    playlist_obj = spotify_client.get_playlist_by_id(source_id)
                    if playlist_obj and playlist_obj.tracks:
                        tracks = []
                        for t in playlist_obj.tracks:
                            artist_name = t.artists[0] if t.artists else ''
                            track_dict = {
                                'track_name': t.name or '',
                                'artist_name': str(artist_name),
                                'album_name': t.album or '',
                                'duration_ms': t.duration_ms or 0,
                                'source_track_id': t.id or '',
                            }
                            # Spotify data IS official — auto-mark as discovered
                            if t.id:
                                track_dict['extra_data'] = json.dumps({
                                    'discovered': True,
                                    'provider': 'spotify',
                                    'confidence': 1.0,
                                    'matched_data': {
                                        'id': t.id,
                                        'name': t.name or '',
                                        'artists': [{'name': str(a)} for a in (t.artists or [])],
                                        'album': t.album or '',
                                        'duration_ms': t.duration_ms or 0,
                                    }
                                })
                            tracks.append(track_dict)

                elif source == 'tidal' and tidal_client and tidal_client.is_authenticated():
                    full_playlist = tidal_client.get_playlist(source_id)
                    if full_playlist and full_playlist.tracks:
                        tracks = []
                        for t in full_playlist.tracks:
                            artist_name = t.artists[0] if t.artists else ''
                            tracks.append({
                                'track_name': t.name or '',
                                'artist_name': str(artist_name),
                                'album_name': t.album or '',
                                'duration_ms': t.duration_ms or 0,
                                'source_track_id': t.id or '',
                            })

                elif source == 'youtube':
                    yt_url = f"https://www.youtube.com/playlist?list={source_id}"
                    playlist_data = parse_youtube_playlist(yt_url)
                    if playlist_data and playlist_data.get('tracks'):
                        tracks = []
                        for t in playlist_data['tracks']:
                            artist_name = t['artists'][0] if t.get('artists') else ''
                            tracks.append({
                                'track_name': t.get('name', ''),
                                'artist_name': str(artist_name),
                                'album_name': '',
                                'duration_ms': t.get('duration_ms', 0),
                                'source_track_id': t.get('id', ''),
                            })

                if tracks is not None:
                    # Compare old vs new track IDs to detect changes
                    old_tracks = db.get_mirrored_playlist_tracks(pl['id']) if pl.get('id') else []
                    old_ids = {t.get('source_track_id') for t in old_tracks if t.get('source_track_id')}
                    new_ids = {t.get('source_track_id') for t in tracks if t.get('source_track_id')}

                    # Preserve existing discovery extra_data for tracks that still exist
                    old_extra_map = db.get_mirrored_tracks_extra_data_map(pl['id']) if pl.get('id') else {}
                    for t in tracks:
                        sid = t.get('source_track_id', '')
                        if sid and sid in old_extra_map and 'extra_data' not in t:
                            t['extra_data'] = old_extra_map[sid]

                    db.mirror_playlist(
                        source=source,
                        source_playlist_id=source_id,
                        name=pl['name'],
                        tracks=tracks,
                        profile_id=pl.get('profile_id', 1),
                        owner=pl.get('owner'),
                        image_url=pl.get('image_url'),
                    )
                    refreshed += 1

                    # Emit playlist_changed if tracks actually changed
                    if old_ids != new_ids:
                        added_count = len(new_ids - old_ids)
                        removed_count = len(old_ids - new_ids)
                        _update_automation_progress(auto_id,
                            log_line=f'"{pl.get("name", "")}" — {added_count} added, {removed_count} removed', log_type='success')
                        try:
                            if automation_engine:
                                automation_engine.emit('playlist_changed', {
                                    'playlist_name': pl.get('name', ''),
                                    'old_count': str(len(old_ids)),
                                    'new_count': str(len(new_ids)),
                                    'added': str(added_count),
                                    'removed': str(removed_count),
                                })
                        except Exception:
                            pass
                    else:
                        _update_automation_progress(auto_id,
                            log_line=f'No changes: "{pl.get("name", "")}"', log_type='skip')
            except Exception as e:
                errors.append(f"{pl.get('name', '?')}: {str(e)}")
                _update_automation_progress(auto_id,
                    log_line=f'Error: {pl.get("name", "?")} — {str(e)}', log_type='error')
        return {'status': 'completed', 'refreshed': str(refreshed), 'errors': str(len(errors))}

    def _auto_sync_playlist(config):
        """Sync a mirrored playlist to media server.
        Uses discovered metadata when available, skips undiscovered tracks."""
        auto_id = config.get('_automation_id')
        playlist_id = config.get('playlist_id')
        if not playlist_id:
            return {'status': 'error', 'reason': 'No playlist specified'}

        db = get_database()
        pl = db.get_mirrored_playlist(int(playlist_id))
        if not pl:
            return {'status': 'error', 'reason': 'Playlist not found'}

        tracks = db.get_mirrored_playlist_tracks(int(playlist_id))
        if not tracks:
            return {'status': 'error', 'reason': 'No tracks in playlist'}

        # Convert mirrored tracks to format expected by _run_sync_task
        # Use discovered metadata when available, skip undiscovered tracks
        tracks_json = []
        skipped_count = 0

        for t in tracks:
            # Parse extra_data for discovery info
            extra = {}
            if t.get('extra_data'):
                try:
                    extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data']
                except (json.JSONDecodeError, TypeError):
                    pass

            if extra.get('discovered') and extra.get('matched_data'):
                # Use official discovered metadata
                md = extra['matched_data']
                tracks_json.append({
                    'name': md.get('name', ''),
                    'artists': md.get('artists', [{'name': t.get('artist_name', '')}]),
                    'album': md.get('album', ''),
                    'duration_ms': md.get('duration_ms', 0),
                    'id': md.get('id', ''),
                })
            else:
                # NOT discovered — skip to prevent garbage in wishlist
                skipped_count += 1

        _update_automation_progress(auto_id, progress=50,
            phase=f'Syncing "{pl["name"]}"',
            log_line=f'{len(tracks_json)} discovered, {skipped_count} skipped',
            log_type='info' if tracks_json else 'skip')

        if not tracks_json:
            _update_automation_progress(auto_id,
                log_line=f'No discovered tracks — {skipped_count} need discovery first', log_type='skip')
            return {
                'status': 'skipped',
                'reason': f'No discovered tracks to sync ({skipped_count} tracks need discovery first)',
                'skipped_tracks': str(skipped_count),
            }

        sync_id = f"auto_mirror_{playlist_id}"
        _update_automation_progress(auto_id, progress=90,
            log_line=f'Starting sync: {len(tracks_json)} tracks', log_type='success')
        threading.Thread(
            target=_run_sync_task,
            args=(sync_id, pl['name'], json.dumps(tracks_json)),
            daemon=True,
            name=f'auto-sync-{playlist_id}'
        ).start()
        return {
            'status': 'started',
            'playlist_name': pl['name'],
            'discovered_tracks': str(len(tracks_json)),
            'skipped_tracks': str(skipped_count),
        }

    def _auto_discover_playlist(config):
        """Discover official Spotify/iTunes metadata for mirrored playlist tracks."""
        db = get_database()
        playlist_id = config.get('playlist_id')
        discover_all = config.get('all', False)

        if discover_all:
            playlists = db.get_mirrored_playlists()
        elif playlist_id:
            p = db.get_mirrored_playlist(int(playlist_id))
            playlists = [p] if p else []
        else:
            return {'status': 'error', 'reason': 'No playlist specified'}

        if not playlists:
            return {'status': 'error', 'reason': 'No playlists found'}

        threading.Thread(
            target=_run_playlist_discovery_worker,
            args=(playlists, config.get('_automation_id')),
            daemon=True,
            name='auto-discover-playlist'
        ).start()
        names = ', '.join(p['name'] for p in playlists[:3])
        return {'status': 'started', 'playlist_count': str(len(playlists)), 'playlists': names,
                '_manages_own_progress': True}

    automation_engine.register_action_handler('refresh_mirrored', _auto_refresh_mirrored)
    automation_engine.register_action_handler('sync_playlist', _auto_sync_playlist)
    automation_engine.register_action_handler('discover_playlist', _auto_discover_playlist)

    # --- Phase 3 action handlers ---

    def _auto_start_database_update(config):
        if db_update_state.get('status') == 'running':
            return {'status': 'skipped', 'reason': 'Database update already running'}
        full = config.get('full_refresh', False)
        active_server = config_manager.get_active_media_server()
        with db_update_lock:
            db_update_state.update({
                "status": "running", "phase": "Initializing...",
                "progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": ""
            })
        db_update_executor.submit(_run_db_update_task, full, active_server)
        return {'status': 'started', 'full_refresh': str(full)}

    def _auto_run_duplicate_cleaner(config):
        if duplicate_cleaner_state.get('status') == 'running':
            return {'status': 'skipped', 'reason': 'Duplicate cleaner already running'}
        duplicate_cleaner_executor.submit(_run_duplicate_cleaner)
        return {'status': 'started'}

    def _auto_clear_quarantine(config):
        import shutil as _shutil
        quarantine_path = os.path.join(docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')), 'ss_quarantine')
        if not os.path.exists(quarantine_path):
            return {'status': 'completed', 'removed': '0'}
        removed = 0
        for f in os.listdir(quarantine_path):
            fp = os.path.join(quarantine_path, f)
            try:
                if os.path.isfile(fp):
                    os.remove(fp)
                    removed += 1
                elif os.path.isdir(fp):
                    _shutil.rmtree(fp)
                    removed += 1
            except Exception:
                pass
        return {'status': 'completed', 'removed': str(removed)}

    def _auto_cleanup_wishlist(config):
        db = get_database()
        removed = db.remove_wishlist_duplicates(get_current_profile_id())
        return {'status': 'completed', 'removed': str(removed or 0)}

    def _auto_update_discovery_pool(config):
        try:
            from core.watchlist_scanner import get_watchlist_scanner
            scanner = get_watchlist_scanner(spotify_client)
            threading.Thread(target=scanner.update_discovery_pool_incremental,
                             args=(get_current_profile_id(),), daemon=True).start()
            return {'status': 'started'}
        except Exception as e:
            return {'status': 'error', 'reason': str(e)}

    def _auto_start_quality_scan(config):
        if quality_scanner_state.get('status') == 'running':
            return {'status': 'skipped', 'reason': 'Quality scan already running'}
        scope = config.get('scope', 'watchlist')
        quality_scanner_executor.submit(_run_quality_scanner, scope, get_current_profile_id())
        return {'status': 'started', 'scope': scope}

    def _auto_backup_database(config):
        import sqlite3, glob as _glob
        db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db')
        if not os.path.exists(db_path):
            return {'status': 'error', 'reason': 'Database file not found'}
        max_backups = 3
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        backup_path = f"{db_path}.backup_{timestamp}"
        # Use SQLite backup API for safe hot-copy of active database
        src = sqlite3.connect(db_path)
        dst = sqlite3.connect(backup_path)
        src.backup(dst)
        dst.close()
        src.close()
        size_mb = round(os.path.getsize(backup_path) / (1024 * 1024), 1)
        # Rolling cleanup — keep only the newest N backups
        existing = sorted(_glob.glob(f"{db_path}.backup_*"), key=os.path.getmtime)
        while len(existing) > max_backups:
            try:
                os.remove(existing.pop(0))
            except Exception:
                pass
        return {'status': 'completed', 'backup_path': backup_path, 'size_mb': str(size_mb)}

    automation_engine.register_action_handler('start_database_update', _auto_start_database_update,
                                              lambda: db_update_state.get('status') == 'running')
    automation_engine.register_action_handler('run_duplicate_cleaner', _auto_run_duplicate_cleaner,
                                              lambda: duplicate_cleaner_state.get('status') == 'running')
    automation_engine.register_action_handler('clear_quarantine', _auto_clear_quarantine)
    automation_engine.register_action_handler('cleanup_wishlist', _auto_cleanup_wishlist)
    automation_engine.register_action_handler('update_discovery_pool', _auto_update_discovery_pool)
    automation_engine.register_action_handler('start_quality_scan', _auto_start_quality_scan,
                                              lambda: quality_scanner_state.get('status') == 'running')
    automation_engine.register_action_handler('backup_database', _auto_backup_database)

    # Register progress tracking callbacks
    def _progress_init(aid, name, action_type):
        _init_automation_progress(aid, name, action_type)

    def _progress_finish(aid, result):
        result_status = result.get('status', '')
        # Skip for handlers that manage their own progress lifecycle
        # (they call _update_automation_progress(status='finished') themselves)
        if result.get('_manages_own_progress'):
            return
        status = 'error' if result_status == 'error' else 'finished'
        msg = result.get('error', result.get('reason', result_status or 'done'))
        _update_automation_progress(aid, status=status, progress=100,
                                     phase='Error' if status == 'error' else 'Complete',
                                     log_line=msg, log_type='error' if status == 'error' else 'success')

    automation_engine.register_progress_callbacks(_progress_init, _progress_finish)

    print("✅ Automation action handlers registered")


def _emit_track_downloaded(context):
    """Emit track_downloaded event for automation engine. Safe to call anywhere."""
    try:
        if not automation_engine:
            return
        ti = context.get('track_info') or context.get('search_result') or {}
        artist_name = ''
        artists = ti.get('artists', [])
        if artists:
            a = artists[0]
            artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a)
        automation_engine.emit('track_downloaded', {
            'artist': artist_name,
            'title': ti.get('name', ti.get('title', '')),
            'album': ti.get('album', ''),
            'quality': context.get('_audio_quality', 'Unknown'),
        })
    except Exception:
        pass

# --- Register Public REST API Blueprint (v1) ---
try:
    from api import create_api_blueprint, limiter
    limiter.init_app(app)
    api_bp = create_api_blueprint()
    app.register_blueprint(api_bp, url_prefix='/api/v1')
    app.soulsync = {
        'spotify_client': spotify_client,
        'soulseek_client': soulseek_client,
        'tidal_client': tidal_client,
        'matching_engine': matching_engine,
        'config_manager': config_manager,
        'hydrabase_client': None,       # updated after Hydrabase init
        'hydrabase_worker': None,       # updated after Hydrabase init
    }
    print("✅ Public REST API v1 registered at /api/v1")
except Exception as e:
    print(f"⚠️ Public REST API v1 failed to register: {e}")

# --- Global Streaming State Management ---
# Thread-safe state tracking for streaming functionality
stream_state = {
    "status": "stopped",  # States: stopped, loading, queued, ready, error
    "progress": 0,
    "track_info": None,
    "file_path": None,    # Path to the audio file in the 'Stream' folder
    "error_message": None
}
stream_lock = threading.Lock()  # Prevent race conditions
stream_background_task = None
stream_executor = ThreadPoolExecutor(max_workers=1)  # Only one stream at a time

# --- Global OAuth State Management ---
# Store PKCE values for Tidal OAuth flow
tidal_oauth_state = {
    "code_verifier": None,
    "code_challenge": None
}
tidal_oauth_lock = threading.Lock()

db_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DBUpdate")
db_update_worker = None
db_update_state = {
    "status": "idle",  # idle, running, finished, error
    "phase": "Idle",
    "progress": 0,
    "current_item": "",
    "processed": 0,
    "total": 0,
    "error_message": "",
    "removed_artists": 0,
    "removed_albums": 0,
    "removed_tracks": 0
}

# Quality Scanner state
quality_scanner_state = {
    "status": "idle",  # idle, running, finished, error
    "phase": "Ready to scan",
    "progress": 0,
    "processed": 0,
    "total": 0,
    "quality_met": 0,
    "low_quality": 0,
    "matched": 0,
    "error_message": "",
    "results": []  # List of low quality tracks with match status
}
quality_scanner_lock = threading.Lock()
quality_scanner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="QualityScanner")

# Duplicate Cleaner state
duplicate_cleaner_state = {
    "status": "idle",  # idle, running, finished, error
    "phase": "Ready to scan",
    "progress": 0,
    "files_scanned": 0,
    "total_files": 0,
    "duplicates_found": 0,
    "deleted": 0,
    "space_freed": 0,  # in bytes
    "error_message": ""
}
duplicate_cleaner_lock = threading.Lock()
duplicate_cleaner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DuplicateCleaner")

# --- Retag Tool Globals ---
retag_state = {
    "status": "idle",
    "phase": "Ready",
    "progress": 0,
    "current_track": "",
    "total_tracks": 0,
    "processed": 0,
    "error_message": ""
}
retag_lock = threading.Lock()
retag_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="RetagWorker")

# --- Sync Page Globals ---
sync_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="SyncWorker")
active_sync_workers = {}  # Key: playlist_id, Value: Future object
sync_states = {}          # Key: playlist_id, Value: dict with progress info
sync_lock = threading.Lock()
db_update_lock = threading.Lock()

# --- Automation Progress Tracking ---
automation_progress_states = {}   # automation_id (int) -> state dict
automation_progress_lock = threading.Lock()

def _init_automation_progress(automation_id, automation_name, action_type):
    """Initialize progress state when an automation starts running."""
    with automation_progress_lock:
        automation_progress_states[automation_id] = {
            'status': 'running',
            'action_type': action_type,
            'progress': 0, 'phase': 'Starting...', 'current_item': '',
            'processed': 0, 'total': 0,
            'log': [{'type': 'info', 'text': f'Starting {automation_name}'}],
            'started_at': datetime.now().isoformat(),
            'finished_at': None,
        }

def _update_automation_progress(automation_id, **kwargs):
    """Update progress state from handler threads. Thread-safe."""
    if automation_id is None:
        return
    with automation_progress_lock:
        state = automation_progress_states.get(automation_id)
        if not state:
            return
        for k, v in kwargs.items():
            if k == 'log_line':
                state['log'].append({'type': kwargs.get('log_type', 'info'), 'text': v})
                if len(state['log']) > 50:
                    state['log'] = state['log'][-50:]
            elif k != 'log_type':
                state[k] = v
        # Immediate emit on finish so frontend gets final state without waiting for loop
        if kwargs.get('status') in ('finished', 'error'):
            state['finished_at'] = datetime.now().isoformat()
            try:
                socketio.emit('automation:progress', {str(automation_id): dict(state)})
            except Exception:
                pass

# --- Global Matched Downloads Context Management ---
# Thread-safe storage for matched download contexts
# Key: slskd download ID, Value: dict containing Spotify artist/album data
matched_downloads_context = {}
matched_context_lock = threading.Lock()
_orphaned_download_keys = set()  # Context keys of downloads abandoned during retry

# --- File-Level Metadata Write Locking ---
# Prevents concurrent threads from writing metadata to the same file simultaneously
_metadata_write_locks = {}  # file_path -> threading.Lock()
_metadata_locks_lock = threading.Lock()  # Lock for the locks dict

def _get_file_lock(file_path):
    """Get or create a lock for a specific file path to prevent concurrent metadata writes."""
    with _metadata_locks_lock:
        if file_path not in _metadata_write_locks:
            _metadata_write_locks[file_path] = threading.Lock()
        return _metadata_write_locks[file_path]

# --- Download Missing Tracks Modal State Management ---
# Thread-safe state tracking for modal download functionality with batch management
missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker")
download_tasks = {}  # task_id -> task state dict
download_batches = {}  # batch_id -> {queue, active_count, max_concurrent}
tasks_lock = threading.Lock()
batch_locks = {}  # batch_id -> Lock() for atomic batch operations

# --- Session Download Statistics ---
# Track individual download completions (matches dashboard.py behavior)
session_completed_downloads = 0
session_stats_lock = threading.Lock()

def _mark_task_completed(task_id, track_info=None):
    """
    Mark a download task as completed and increment session counter.
    Centralizes completion logic to ensure consistent behavior.
    Assumes task_id exists in download_tasks (should be called within tasks_lock).
    """
    global session_completed_downloads

    download_tasks[task_id]['status'] = 'completed'

    # Increment session counter (matches dashboard.py behavior)
    with session_stats_lock:
        session_completed_downloads += 1

# --- Automatic Wishlist Processing Infrastructure ---
# Server-side timer system for automatic wishlist processing (replaces client-side JavaScript timers)
# --- Automatic Wishlist/Watchlist Processing Flags ---
# Processing state flags (guards/recovery — timers are now managed by AutomationEngine)
wishlist_auto_processing = False  # Flag to prevent concurrent auto-processing
wishlist_auto_processing_timestamp = 0  # Timestamp when processing started (for stuck detection)
wishlist_timer_lock = threading.Lock()  # Thread safety for flag operations

watchlist_auto_scanning = False  # Flag to prevent concurrent auto-scanning
watchlist_auto_scanning_timestamp = 0  # Timestamp when scanning started (for stuck detection)
watchlist_timer_lock = threading.Lock()  # Thread safety for flag operations

# --- Shared Transfer Data Cache ---
# Cache transfer data to avoid hammering the Soulseek API with multiple concurrent modals
transfer_data_cache = {
    'data': {},
    'last_update': 0,
    'update_lock': threading.Lock(),
    'cache_duration': 0.75  # Cache for 0.75 seconds for faster updates
}

def get_cached_transfer_data():
    """
    Get transfer data with caching to reduce API calls when multiple modals are active.
    Returns a lookup dictionary for efficient transfer matching.
    """
    current_time = time.time()
    
    with transfer_data_cache['update_lock']:
        # Check if cache is still valid
        if (current_time - transfer_data_cache['last_update']) < transfer_data_cache['cache_duration']:
            return transfer_data_cache['data']
        
        # Cache expired or empty, fetch new data
        live_transfers_lookup = {}
        try:
            # First, get Soulseek downloads from API
            transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
            if transfers_data:
                all_transfers = []
                for user_data in transfers_data:
                    username = user_data.get('username', 'Unknown')
                    if 'directories' in user_data:
                        for directory in user_data['directories']:
                            if 'files' in directory:
                                for file_info in directory['files']:
                                    file_info['username'] = username
                                    all_transfers.append(file_info)
                for transfer in all_transfers:
                    key = _make_context_key(transfer.get('username'), transfer.get('filename', ''))
                    live_transfers_lookup[key] = transfer

            # Also add YouTube/Tidal downloads (through orchestrator)
            try:
                all_downloads = run_async(soulseek_client.get_all_downloads())
                for download in all_downloads:
                    # Only add YouTube/Tidal downloads (Soulseek ones are already in the lookup)
                    if download.username in ('youtube', 'tidal'):
                        key = _make_context_key(download.username, download.filename)
                        # Convert DownloadStatus to transfer dict format
                        live_transfers_lookup[key] = {
                            'id': download.id,
                            'filename': download.filename,
                            'username': download.username,
                            'state': download.state,
                            'percentComplete': download.progress,
                            'size': download.size,
                            'bytesTransferred': download.transferred,
                            'averageSpeed': download.speed,
                        }
            except Exception as e:
                print(f"⚠️ Could not fetch YouTube/Tidal downloads: {e}")

            # Update cache
            transfer_data_cache['data'] = live_transfers_lookup
            transfer_data_cache['last_update'] = current_time
            
        except Exception as e:
            print(f"⚠️ Could not fetch live transfers (cached): {e}")
            # Return empty dict on error, but don't update cache timestamp
            # This way we'll retry on the next request
            return {}
    
    return live_transfers_lookup

# --- Beatport Data Cache ---
# Cache Beatport scraping data to reduce load times and avoid hammering Beatport.com
beatport_data_cache = {
    'homepage': {
        'hero_tracks': {'data': None, 'timestamp': 0, 'ttl': 86400},       # 24 hours
        'top_10_lists': {'data': None, 'timestamp': 0, 'ttl': 86400},      # 24 hours
        'top_10_releases': {'data': None, 'timestamp': 0, 'ttl': 86400},   # 24 hours
        'new_releases': {'data': None, 'timestamp': 0, 'ttl': 86400},      # 24 hours
        'hype_picks': {'data': None, 'timestamp': 0, 'ttl': 86400},        # 24 hours
        'featured_charts': {'data': None, 'timestamp': 0, 'ttl': 86400},   # 24 hours
        'dj_charts': {'data': None, 'timestamp': 0, 'ttl': 86400}          # 24 hours
    },
    'genre': {
        # Future expansion for genre-specific caching
        # 'house': {'top_10': {...}, 'releases': {...}},
        # 'techno': {'top_10': {...}, 'releases': {...}}
    },
    'cache_lock': threading.Lock()
}

def get_cached_beatport_data(section_type, data_key, genre_slug=None):
    """
    Get Beatport data from cache if valid, otherwise return None.

    Args:
        section_type: 'homepage' or 'genre'
        data_key: specific data type (e.g., 'hero_tracks', 'top_10_lists')
        genre_slug: only used for genre section_type

    Returns:
        Cached data if valid, None if cache miss or expired
    """
    current_time = time.time()

    with beatport_data_cache['cache_lock']:
        try:
            if section_type == 'homepage':
                cache_entry = beatport_data_cache['homepage'].get(data_key)
            elif section_type == 'genre' and genre_slug:
                cache_entry = beatport_data_cache['genre'].get(genre_slug, {}).get(data_key)
            else:
                return None

            if not cache_entry:
                return None

            # Check if cache is still valid
            age = current_time - cache_entry['timestamp']
            if age < cache_entry['ttl'] and cache_entry['data'] is not None:
                print(f"🎯 Cache HIT for {section_type}/{data_key} (age: {age:.1f}s)")
                return cache_entry['data']
            else:
                print(f"⏰ Cache MISS for {section_type}/{data_key} (age: {age:.1f}s, ttl: {cache_entry['ttl']}s)")
                return None

        except Exception as e:
            print(f"⚠️ Cache lookup error for {section_type}/{data_key}: {e}")
            return None

def set_cached_beatport_data(section_type, data_key, data, genre_slug=None):
    """
    Store Beatport data in cache with current timestamp.

    Args:
        section_type: 'homepage' or 'genre'
        data_key: specific data type (e.g., 'hero_tracks', 'top_10_lists')
        data: the data to cache
        genre_slug: only used for genre section_type
    """
    current_time = time.time()

    with beatport_data_cache['cache_lock']:
        try:
            if section_type == 'homepage':
                if data_key in beatport_data_cache['homepage']:
                    beatport_data_cache['homepage'][data_key]['data'] = data
                    beatport_data_cache['homepage'][data_key]['timestamp'] = current_time
                    print(f"💾 Cached {section_type}/{data_key} (ttl: {beatport_data_cache['homepage'][data_key]['ttl']}s)")
            elif section_type == 'genre' and genre_slug:
                # Initialize genre cache if not exists
                if genre_slug not in beatport_data_cache['genre']:
                    beatport_data_cache['genre'][genre_slug] = {}

                # For genre caching, we need to define TTL structure (for future use)
                if data_key not in beatport_data_cache['genre'][genre_slug]:
                    beatport_data_cache['genre'][genre_slug][data_key] = {
                        'data': None, 'timestamp': 0, 'ttl': 600  # Default 10 minutes
                    }

                beatport_data_cache['genre'][genre_slug][data_key]['data'] = data
                beatport_data_cache['genre'][genre_slug][data_key]['timestamp'] = current_time
                print(f"💾 Cached {section_type}/{genre_slug}/{data_key}")

        except Exception as e:
            print(f"⚠️ Cache storage error for {section_type}/{data_key}: {e}")

def add_cache_headers(response, cache_duration=300):
    """
    Add HTTP cache control headers to responses for browser-side caching.

    Args:
        response: Flask response object
        cache_duration: Cache duration in seconds (default: 5 minutes)
    """
    response.headers['Cache-Control'] = f'public, max-age={cache_duration}'
    response.headers['Pragma'] = 'cache'
    return response

# --- Background Download Monitoring (GUI Parity) ---
class WebUIDownloadMonitor:
    """
    Background monitor for download progress and retry logic, matching GUI's SyncStatusProcessingWorker.
    Implements identical timeout detection and automatic retry functionality.
    """
    def __init__(self):
        self.monitoring = False
        self.monitor_thread = None
        self.monitored_batches = set()
        
    def start_monitoring(self, batch_id):
        """Start monitoring a download batch"""
        self.monitored_batches.add(batch_id)
        if not self.monitoring:
            self.monitoring = True
            self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True)
            self.monitor_thread.start()
            print(f"🔍 Started download monitor for batch {batch_id}")
    
    def stop_monitoring(self, batch_id):
        """Stop monitoring a specific batch"""
        self.monitored_batches.discard(batch_id)
        if not self.monitored_batches:
            self.monitoring = False
            print(f"🛑 Stopped download monitor (no active batches)")
    
    def _monitor_loop(self):
        """Main monitoring loop - checks downloads every 1 second for responsive web UX"""
        while self.monitoring and self.monitored_batches:
            try:
                self._check_all_downloads()
                time.sleep(1)  # 1-second polling for fast web UI updates
            except Exception as e:
                # If we get shutdown errors, stop monitoring gracefully
                if "interpreter shutdown" in str(e) or "cannot schedule new futures" in str(e):
                    print(f"🛑 Monitor detected shutdown, stopping gracefully")
                    self.monitoring = False
                    break
                print(f"❌ Download monitor error: {e}")
                
        print(f"🔍 Download monitor loop ended")
    
    def _check_all_downloads(self):
        """Check all active downloads for timeouts and failures"""
        current_time = time.time()

        # Get live transfer data from slskd
        live_transfers_lookup = self._get_live_transfers()

        # Track tasks with exhausted retries to handle after releasing lock
        exhausted_tasks = []  # List of (batch_id, task_id) tuples
        # Track completed downloads to handle after releasing lock (prevents deadlock)
        completed_tasks = []  # List of (batch_id, task_id) tuples
        # Track deferred operations (network calls, nested locks) to run after releasing tasks_lock
        deferred_ops = []

        with tasks_lock:
            # Check all monitored batches for timeouts and errors
            for batch_id in list(self.monitored_batches):
                if batch_id not in download_batches:
                    self.monitored_batches.discard(batch_id)
                    continue

                for task_id in download_batches[batch_id].get('queue', []):
                    task = download_tasks.get(task_id)
                    if not task or task['status'] not in ['downloading', 'queued']:
                        continue

                    # Check for timeouts and errors - retries handled directly in _should_retry_task
                    # If _should_retry_task returns True, it means retries were exhausted
                    retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops)
                    # Collect exhausted tasks to handle outside lock (prevents deadlock)
                    if retry_exhausted:
                        exhausted_tasks.append((batch_id, task_id))

                    # ENHANCED: Check for successful completions (especially YouTube)
                    task_filename = task.get('filename') or task.get('track_info', {}).get('filename')
                    task_username = task.get('username') or task.get('track_info', {}).get('username')

                    if task_filename and task_username:
                        lookup_key = _make_context_key(task_username, task_filename)
                        live_info = live_transfers_lookup.get(lookup_key)

                        if live_info:
                            state = live_info.get('state', '')
                            # Trigger post-processing if download is completed successfully
                            # slskd uses compound states like 'Completed, Succeeded' - use substring matching
                            # Must exclude error states first (matching _build_batch_status_data's prioritized checking)
                            has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state)
                            has_completion = ('Completed' in state or 'Succeeded' in state)
                            # Verify bytes actually transferred before trusting state string.
                            # slskd can report "Completed" before the full file is flushed to disk,
                            # or on connection drops that leave a partial file.
                            if has_completion and not has_error:
                                expected_size = live_info.get('size', 0)
                                transferred = live_info.get('bytesTransferred', 0)
                                if expected_size > 0 and transferred < expected_size:
                                    if not task.get('_incomplete_warned'):
                                        print(f"⚠️ Monitor: {task_id} state={state} but bytes incomplete ({transferred}/{expected_size}) — waiting")
                                        task['_incomplete_warned'] = True
                                    continue
                            if has_completion and not has_error and task['status'] == 'downloading':
                                task.pop('_incomplete_warned', None)
                                # CRITICAL FIX: Transition to 'post_processing' HERE so downloads
                                # don't depend on browser polling to trigger post-processing.
                                # Previously, post-processing was only submitted by _build_batch_status_data
                                # (called from browser-polled endpoints), meaning closing the browser
                                # left tasks stuck in 'downloading' forever.
                                task['status'] = 'post_processing'
                                task['status_change_time'] = current_time
                                print(f"✅ Monitor detected completed download for {task_id} ({state}) - submitting post-processing")
                                # Collect for handling outside the lock to prevent deadlock.
                                # _on_download_completed acquires tasks_lock which is non-reentrant.
                                completed_tasks.append((batch_id, task_id))

        # ---- All work below runs WITHOUT tasks_lock held ----

        # Execute deferred operations from _should_retry_task (network calls, nested locks)
        for op in deferred_ops:
            try:
                if op[0] == 'cancel_download':
                    _, download_id, username = op
                    print(f"🚫 [Deferred] Cancelling download: {download_id} from {username}")
                    run_async(soulseek_client.cancel_download(download_id, username, remove=True))
                    print(f"✅ [Deferred] Successfully cancelled download {download_id}")
                elif op[0] == 'cleanup_orphan':
                    _, context_key = op
                    with matched_context_lock:
                        matched_downloads_context.pop(context_key, None)
                    print(f"🧹 [Deferred] Cleaned up orphaned download context: {context_key}")
                elif op[0] == 'restart_worker':
                    _, task_id, batch_id = op
                    print(f"🚀 [Deferred] Restarting worker for task {task_id}")
                    missing_download_executor.submit(_download_track_worker, task_id, batch_id)
                    print(f"✅ [Deferred] Successfully restarted worker for task {task_id}")
            except Exception as e:
                print(f"⚠️ [Deferred] Error executing deferred operation {op[0]}: {e}")

        # Handle completed downloads outside the lock to prevent deadlock
        # (_on_download_completed acquires tasks_lock internally)
        for batch_id, task_id in completed_tasks:
            try:
                # Submit post-processing worker (file move, tagging, AcoustID verification)
                # This makes batch downloads fully independent of browser polling.
                print(f"✅ [Monitor] Submitting post-processing worker for task {task_id}")
                missing_download_executor.submit(_run_post_processing_worker, task_id, batch_id)
                # Chain to next download in the batch queue
                _on_download_completed(batch_id, task_id, success=True)
            except Exception as e:
                print(f"❌ [Monitor] Error handling completed task {task_id}: {e}")
        # Handle exhausted retry tasks outside the lock to prevent deadlock
        for batch_id, task_id in exhausted_tasks:
            try:
                print(f"📋 [Monitor] Calling completion callback for exhausted task {task_id}")
                _on_download_completed(batch_id, task_id, success=False)
            except Exception as e:
                print(f"❌ [Monitor] Error handling exhausted task {task_id}: {e}")

        # ENHANCED: Add worker count validation to detect ghost workers
        self._validate_worker_counts()
    
    def _get_live_transfers(self):
        """Get current transfer status from slskd API and YouTube client"""
        try:
            # Check if we should stop due to shutdown
            if not self.monitoring:
                return {}

            live_transfers = {}

            # Get Soulseek downloads from API
            transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
            if transfers_data:
                for user_data in transfers_data:
                    username = user_data.get('username', 'Unknown')
                    if 'directories' in user_data:
                        for directory in user_data['directories']:
                            if 'files' in directory:
                                for file_info in directory['files']:
                                    key = _make_context_key(username, file_info.get('filename', ''))
                                    live_transfers[key] = file_info

            # Also get YouTube/Tidal downloads (through orchestrator)
            try:
                all_downloads = run_async(soulseek_client.get_all_downloads())
                for download in all_downloads:
                    # Only add YouTube/Tidal downloads (Soulseek ones are already in the lookup)
                    if download.username in ('youtube', 'tidal'):
                        key = _make_context_key(download.username, download.filename)
                        # Convert DownloadStatus to transfer dict format for monitor compatibility
                        live_transfers[key] = {
                            'id': download.id,
                            'filename': download.filename,
                            'username': download.username,
                            'state': download.state,
                            'percentComplete': download.progress,
                            'size': download.size,
                            'bytesTransferred': download.transferred,
                            'averageSpeed': download.speed,
                        }
            except Exception as yt_error:
                print(f"⚠️ Monitor: Could not fetch YouTube/Tidal downloads: {yt_error}")

            return live_transfers
        except Exception as e:
            # If we get shutdown-related errors, stop monitoring immediately
            if ("interpreter shutdown" in str(e) or 
                "cannot schedule new futures" in str(e) or
                "Event loop is closed" in str(e)):
                print(f"🛑 Monitor detected shutdown, stopping immediately")
                self.monitoring = False
                return {}
            else:
                print(f"⚠️ Monitor: Could not fetch live transfers: {e}")
            return {}
    
    def _should_retry_task(self, task_id, task, live_transfers_lookup, current_time, deferred_ops):
        """
        Determine if a task should be retried due to timeout (matches GUI logic).

        IMPORTANT: This runs while tasks_lock is held. All network calls (slskd API)
        and nested lock acquisitions (matched_context_lock) are collected into deferred_ops
        to be executed AFTER releasing tasks_lock. This prevents deadlocks and long lock holds.

        Returns True if retries are exhausted and _on_download_completed should be called outside the lock.
        """
        ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {}
        task_filename = task.get('filename') or ti.get('filename')
        task_username = task.get('username') or ti.get('username')

        if not task_filename or not task_username:
            return False

        lookup_key = _make_context_key(task_username, task_filename)
        live_info = live_transfers_lookup.get(lookup_key)

        if not live_info:
            # Task not in live transfers but status is downloading/queued - likely stuck
            if current_time - task.get('status_change_time', current_time) > 90:
                retry_count = task.get('stuck_retry_count', 0)
                last_retry = task.get('last_retry_time', 0)

                if retry_count < 3 and (current_time - last_retry) > 30:
                    print(f"⚠️ Task not in live transfers for >90s - retry {retry_count + 1}/3")
                    task['stuck_retry_count'] = retry_count + 1
                    task['last_retry_time'] = current_time

                    download_id = task.get('download_id')

                    # Defer slskd cancel to outside the lock
                    if task_username and download_id:
                        deferred_ops.append(('cancel_download', download_id, task_username))

                    # Mark current source as used (full filename to match worker format)
                    if task_username and task_filename:
                        used_sources = task.get('used_sources', set())
                        source_key = f"{task_username}_{task_filename}"
                        used_sources.add(source_key)
                        task['used_sources'] = used_sources
                        print(f"🚫 Marked missing-transfer source as used: {source_key}")

                    # Defer orphan cleanup
                    if task_username and task_filename:
                        _orphaned_download_keys.add(lookup_key)
                        deferred_ops.append(('cleanup_orphan', lookup_key))

                    # Clear download info and reset for retry
                    task.pop('download_id', None)
                    task.pop('username', None)
                    task.pop('filename', None)
                    task['status'] = 'searching'
                    task.pop('queued_start_time', None)
                    task.pop('downloading_start_time', None)
                    task['status_change_time'] = current_time
                    print(f"🔄 Task {task.get('track_info', {}).get('name', 'Unknown')} reset for missing-transfer retry")

                    batch_id = task.get('batch_id')
                    if task_id and batch_id:
                        deferred_ops.append(('restart_worker', task_id, batch_id))
                    return False
                elif retry_count < 3:
                    return False
                else:
                    track_label = task.get('track_info', {}).get('name', 'Unknown')
                    tried_sources = task.get('used_sources', set())
                    sources_str = f' (tried {len(tried_sources)} source{"s" if len(tried_sources) != 1 else ""})' if tried_sources else ''
                    print(f"❌ Task failed after 3 retry attempts (not in live transfers)")
                    task['status'] = 'failed'
                    task['error_message'] = f'Download disappeared from transfer list 3 times for "{track_label}"{sources_str} — source may be unavailable'

                    batch_id = task.get('batch_id')
                    if batch_id:
                        return True
                    return False
            return False

        state_str = live_info.get('state', '')
        progress = live_info.get('percentComplete', 0)

        # IMMEDIATE ERROR RETRY: Check for errored/rejected/timed-out downloads first (no timeout needed)
        if 'Errored' in state_str or 'Failed' in state_str or 'Rejected' in state_str or 'TimedOut' in state_str:
            retry_count = task.get('error_retry_count', 0)
            last_retry = task.get('last_error_retry_time', 0)

            # Don't retry too frequently (wait at least 5 seconds between error retries)
            if retry_count < 3 and (current_time - last_retry) > 5:  # Max 3 error retry attempts
                print(f"🚨 Task errored (state: {state_str}) - immediate retry {retry_count + 1}/3")
                task['error_retry_count'] = retry_count + 1
                task['last_error_retry_time'] = current_time

                _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {}
                username = task.get('username') or _ti.get('username')
                filename = task.get('filename') or _ti.get('filename')
                download_id = task.get('download_id')

                # Defer slskd cancel to outside the lock
                if username and download_id:
                    deferred_ops.append(('cancel_download', download_id, username))

                # Mark current source as used to prevent retry loops
                # CRITICAL: Use full filename (not basename) to match worker's source_key format
                if username and filename:
                    used_sources = task.get('used_sources', set())
                    source_key = f"{username}_{filename}"
                    used_sources.add(source_key)
                    task['used_sources'] = used_sources
                    print(f"🚫 Marked errored source as used: {source_key}")

                # Defer orphan cleanup to outside the lock (needs matched_context_lock)
                if username and filename:
                    old_context_key = _make_context_key(username, filename)
                    _orphaned_download_keys.add(old_context_key)
                    deferred_ops.append(('cleanup_orphan', old_context_key))

                # Clear download info since we cancelled it
                task.pop('download_id', None)
                task.pop('username', None)
                task.pop('filename', None)

                # Reset task state for immediate retry
                task['status'] = 'searching'
                task.pop('queued_start_time', None)
                task.pop('downloading_start_time', None)
                task['status_change_time'] = current_time
                print(f"🔄 Task {task.get('track_info', {}).get('name', 'Unknown')} reset for error retry")

                # Defer worker restart to outside the lock
                batch_id = task.get('batch_id')
                if task_id and batch_id:
                    deferred_ops.append(('restart_worker', task_id, batch_id))
                return False
            elif retry_count < 3:
                # Wait a bit before next error retry
                return False
            else:
                # Too many error retries, mark as failed
                track_label = task.get('track_info', {}).get('name', 'Unknown')
                tried_sources = task.get('used_sources', set())
                sources_str = f' (tried {len(tried_sources)} source{"s" if len(tried_sources) != 1 else ""})' if tried_sources else ''
                print(f"❌ Task failed after 3 error retry attempts")
                task['status'] = 'failed'
                task['error_message'] = f'Soulseek transfer errored 3 times for "{track_label}"{sources_str} — all sources failed or became unavailable'

                # CRITICAL: Notify batch manager so track is added to permanently_failed_tracks
                batch_id = task.get('batch_id')
                if batch_id:
                    print(f"📋 [Retry Exhausted] Notifying batch manager of permanent failure for task {task_id}")
                    return True  # Signal that we need to call completion outside the lock
                return False

        # Check for queued timeout (90 seconds like GUI)
        elif 'Queued' in state_str or task['status'] == 'queued':
            if 'queued_start_time' not in task:
                task['queued_start_time'] = current_time
                return False
            else:
                queue_time = current_time - task['queued_start_time']

                # Use context-aware timeouts like GUI:
                # - 15 seconds for artist album downloads (streaming context)
                # - 90 seconds for background playlist downloads
                is_streaming_context = task.get('track_info', {}).get('is_album_download', False)
                timeout_threshold = 15.0 if is_streaming_context else 90.0

                if queue_time > timeout_threshold:
                    # Track retry attempts to prevent rapid loops
                    retry_count = task.get('stuck_retry_count', 0)
                    last_retry = task.get('last_retry_time', 0)

                    # Don't retry too frequently (wait at least 30 seconds between retries)
                    if retry_count < 3 and (current_time - last_retry) > 30:  # Max 3 retry attempts
                        print(f"⚠️ Task stuck in queue for {queue_time:.1f}s - immediate retry {retry_count + 1}/3")
                        task['stuck_retry_count'] = retry_count + 1
                        task['last_retry_time'] = current_time

                        _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {}
                        username = task.get('username') or _ti.get('username')
                        filename = task.get('filename') or _ti.get('filename')
                        download_id = task.get('download_id')

                        # Defer slskd cancel to outside the lock
                        if username and download_id:
                            deferred_ops.append(('cancel_download', download_id, username))

                        # UNIFIED RETRY LOGIC: Handle timeout retry exactly like error retry
                        # Mark current source as used to prevent retry loops
                        # CRITICAL: Use full filename (not basename) to match worker's source_key format
                        if username and filename:
                            used_sources = task.get('used_sources', set())
                            source_key = f"{username}_{filename}"
                            used_sources.add(source_key)
                            task['used_sources'] = used_sources
                            print(f"🚫 Marked timeout source as used: {source_key}")

                        # Defer orphan cleanup to outside the lock (needs matched_context_lock)
                        if username and filename:
                            old_context_key = _make_context_key(username, filename)
                            _orphaned_download_keys.add(old_context_key)
                            deferred_ops.append(('cleanup_orphan', old_context_key))

                        # Clear download info since we cancelled it
                        task.pop('download_id', None)
                        task.pop('username', None)
                        task.pop('filename', None)

                        # Reset task state for immediate retry (like error retry)
                        task['status'] = 'searching'
                        task.pop('queued_start_time', None)
                        task.pop('downloading_start_time', None)
                        task['status_change_time'] = current_time
                        print(f"🔄 Task {task.get('track_info', {}).get('name', 'Unknown')} reset for timeout retry")

                        # Defer worker restart to outside the lock
                        batch_id = task.get('batch_id')
                        if task_id and batch_id:
                            deferred_ops.append(('restart_worker', task_id, batch_id))
                        return False
                    elif retry_count < 3:
                        # Wait longer before next retry
                        return False
                    else:
                        # Too many retries, mark as failed
                        track_label = task.get('track_info', {}).get('name', 'Unknown')
                        tried_sources = task.get('used_sources', set())
                        sources_str = f' (tried {len(tried_sources)} source{"s" if len(tried_sources) != 1 else ""})' if tried_sources else ''
                        print(f"❌ Task failed after 3 retry attempts (queue timeout)")
                        task['status'] = 'failed'
                        task['error_message'] = f'Download stayed queued too long 3 times for "{track_label}"{sources_str} — peers may be offline or have full queues'
                        # Clear timers to prevent further retry loops
                        task.pop('queued_start_time', None)
                        task.pop('downloading_start_time', None)

                        # CRITICAL: Notify batch manager so track is added to permanently_failed_tracks
                        batch_id = task.get('batch_id')
                        if batch_id:
                            print(f"📋 [Retry Exhausted] Notifying batch manager of permanent failure for task {task_id}")
                            return True  # Signal that we need to call completion outside the lock
                        return False
                
        # Check for downloading at 0% timeout (90 seconds like GUI) 
        elif 'InProgress' in state_str and progress < 1:
            if 'downloading_start_time' not in task:
                task['downloading_start_time'] = current_time
                return False
            else:
                download_time = current_time - task['downloading_start_time']
                
                # Use context-aware timeouts like GUI:
                # - 15 seconds for artist album downloads (streaming context)
                # - 90 seconds for background playlist downloads
                is_streaming_context = task.get('track_info', {}).get('is_album_download', False)
                timeout_threshold = 15.0 if is_streaming_context else 90.0

                if download_time > timeout_threshold:
                    retry_count = task.get('stuck_retry_count', 0)
                    last_retry = task.get('last_retry_time', 0)
                    
                    # Don't retry too frequently (wait at least 30 seconds between retries)
                    if retry_count < 3 and (current_time - last_retry) > 30:  # Max 3 retry attempts
                        print(f"⚠️ Task stuck at 0% for {download_time:.1f}s - immediate retry {retry_count + 1}/3")
                        task['stuck_retry_count'] = retry_count + 1
                        task['last_retry_time'] = current_time

                        _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {}
                        username = task.get('username') or _ti.get('username')
                        filename = task.get('filename') or _ti.get('filename')
                        download_id = task.get('download_id')

                        # Defer slskd cancel to outside the lock
                        if username and download_id:
                            deferred_ops.append(('cancel_download', download_id, username))

                        # UNIFIED RETRY LOGIC: Handle 0% timeout retry exactly like error retry
                        # Mark current source as used to prevent retry loops
                        # CRITICAL: Use full filename (not basename) to match worker's source_key format
                        if username and filename:
                            used_sources = task.get('used_sources', set())
                            source_key = f"{username}_{filename}"
                            used_sources.add(source_key)
                            task['used_sources'] = used_sources
                            print(f"🚫 Marked 0% progress source as used: {source_key}")

                        # Defer orphan cleanup to outside the lock (needs matched_context_lock)
                        if username and filename:
                            old_context_key = _make_context_key(username, filename)
                            _orphaned_download_keys.add(old_context_key)
                            deferred_ops.append(('cleanup_orphan', old_context_key))

                        # Clear download info since we cancelled it
                        task.pop('download_id', None)
                        task.pop('username', None)
                        task.pop('filename', None)

                        # Reset task state for immediate retry (like error retry)
                        task['status'] = 'searching'
                        task.pop('queued_start_time', None)
                        task.pop('downloading_start_time', None)
                        task['status_change_time'] = current_time
                        print(f"🔄 Task {task.get('track_info', {}).get('name', 'Unknown')} reset for 0% retry")

                        # Defer worker restart to outside the lock
                        batch_id = task.get('batch_id')
                        if task_id and batch_id:
                            deferred_ops.append(('restart_worker', task_id, batch_id))
                        return False
                    elif retry_count < 3:
                        # Wait longer before next retry
                        return False
                    else:
                        track_label = task.get('track_info', {}).get('name', 'Unknown')
                        tried_sources = task.get('used_sources', set())
                        sources_str = f' (tried {len(tried_sources)} source{"s" if len(tried_sources) != 1 else ""})' if tried_sources else ''
                        print(f"❌ Task failed after 3 retry attempts (0% progress timeout)")
                        task['status'] = 'failed'
                        task['error_message'] = f'Download stuck at 0% three times for "{track_label}"{sources_str} — peers may have connection issues'
                        # Clear timers to prevent further retry loops
                        task.pop('queued_start_time', None)
                        task.pop('downloading_start_time', None)
                        
                        # CRITICAL: Notify batch manager so track is added to permanently_failed_tracks
                        batch_id = task.get('batch_id')
                        if batch_id:
                            print(f"📋 [Retry Exhausted] Notifying batch manager of permanent failure for task {task_id}")
                            return True  # Signal that we need to call completion outside the lock
                        return False
        else:
            # Only reset timers if actual byte progress is being made
            bytes_transferred = live_info.get('bytesTransferred', 0)
            if progress >= 1 or bytes_transferred > 0:
                # Real progress happening, reset timers and retry counts
                task.pop('queued_start_time', None)
                task.pop('downloading_start_time', None)
                task.pop('stuck_retry_count', None)
            else:
                # Unknown state with no progress (e.g., "Requested", "Initializing")
                # Treat like 0% stuck — start/keep the downloading timer running
                if 'downloading_start_time' not in task:
                    task['downloading_start_time'] = current_time
                download_time = current_time - task['downloading_start_time']

                # Use context-aware timeouts
                is_streaming_context = task.get('track_info', {}).get('is_album_download', False)
                timeout_threshold = 15.0 if is_streaming_context else 90.0

                if download_time > timeout_threshold:
                    retry_count = task.get('stuck_retry_count', 0)
                    last_retry = task.get('last_retry_time', 0)

                    if retry_count < 3 and (current_time - last_retry) > 30:
                        print(f"⚠️ Task stuck in unknown state '{state_str}' with 0 progress for {download_time:.1f}s - retry {retry_count + 1}/3")
                        task['stuck_retry_count'] = retry_count + 1
                        task['last_retry_time'] = current_time

                        _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {}
                        username = task.get('username') or _ti.get('username')
                        filename = task.get('filename') or _ti.get('filename')
                        download_id = task.get('download_id')

                        if username and download_id:
                            deferred_ops.append(('cancel_download', download_id, username))

                        if username and filename:
                            used_sources = task.get('used_sources', set())
                            source_key = f"{username}_{filename}"
                            used_sources.add(source_key)
                            task['used_sources'] = used_sources
                            print(f"🚫 Marked unknown-state source as used: {source_key}")

                        if username and filename:
                            old_context_key = _make_context_key(username, filename)
                            _orphaned_download_keys.add(old_context_key)
                            deferred_ops.append(('cleanup_orphan', old_context_key))

                        task.pop('download_id', None)
                        task.pop('username', None)
                        task.pop('filename', None)
                        task['status'] = 'searching'
                        task.pop('queued_start_time', None)
                        task.pop('downloading_start_time', None)
                        task['status_change_time'] = current_time

                        batch_id = task.get('batch_id')
                        if task_id and batch_id:
                            deferred_ops.append(('restart_worker', task_id, batch_id))
                        return False
                    elif retry_count >= 3:
                        track_label = task.get('track_info', {}).get('name', 'Unknown')
                        tried_sources = task.get('used_sources', set())
                        sources_str = f' (tried {len(tried_sources)} source{"s" if len(tried_sources) != 1 else ""})' if tried_sources else ''
                        print(f"❌ Task failed after 3 retry attempts (unknown state '{state_str}')")
                        task['status'] = 'failed'
                        task['error_message'] = f'Download stuck in "{state_str}" state 3 times for "{track_label}"{sources_str}'
                        task.pop('queued_start_time', None)
                        task.pop('downloading_start_time', None)

                        batch_id = task.get('batch_id')
                        if batch_id:
                            return True
                        return False

        return False
    
    
    def _validate_worker_counts(self):
        """
        Validate worker counts to detect and fix ghost workers or orphaned tasks.
        This prevents the modal from showing wrong worker counts permanently.
        """
        try:
            batches_needing_workers = []

            with tasks_lock:
                for batch_id in list(self.monitored_batches):
                    if batch_id not in download_batches:
                        continue

                    batch = download_batches[batch_id]
                    reported_active = batch['active_count']
                    max_concurrent = batch['max_concurrent']
                    queue = batch.get('queue', [])
                    queue_index = batch.get('queue_index', 0)

                    # Count actually active tasks based on status
                    actually_active = 0
                    orphaned_tasks = []
                    # Tasks already processed by _on_download_completed should NOT be counted
                    # as active, even if their status hasn't been updated yet (race condition
                    # between stream processor calling _on_download_completed and
                    # _run_post_processing_worker setting status to 'completed')
                    completed_task_ids = batch.get('_completed_task_ids', set())

                    for task_id in queue:
                        if task_id in download_tasks:
                            task_status = download_tasks[task_id]['status']
                            if task_status in ['searching', 'downloading', 'queued', 'post_processing']:
                                if task_id not in completed_task_ids:
                                    actually_active += 1
                            elif task_status in ['failed', 'completed', 'cancelled', 'not_found'] and task_id in queue[queue_index:]:
                                # These are orphaned tasks - they're done but still in active queue
                                orphaned_tasks.append(task_id)

                    # Check for discrepancies
                    if reported_active != actually_active or orphaned_tasks:
                        print(f"🔍 [Worker Validation] Batch {batch_id}: reported={reported_active}, actual={actually_active}, orphaned={len(orphaned_tasks)}")

                        if orphaned_tasks:
                            print(f"🧹 [Worker Validation] Found {len(orphaned_tasks)} orphaned tasks to cleanup")

                        # Fix the active count if it's wrong
                        if reported_active != actually_active:
                            old_count = batch['active_count']
                            batch['active_count'] = actually_active
                            print(f"✅ [Worker Validation] Fixed active count: {old_count} → {actually_active}")

                            # Defer starting workers to outside the lock
                            if actually_active < max_concurrent and queue_index < len(queue):
                                batches_needing_workers.append(batch_id)

            # Start replacement workers outside the lock
            for batch_id in batches_needing_workers:
                try:
                    print(f"🔄 [Worker Validation] Starting replacement workers for {batch_id}")
                    _start_next_batch_of_downloads(batch_id)
                except Exception as e:
                    print(f"❌ [Worker Validation] Error starting workers for {batch_id}: {e}")

        except Exception as validation_error:
            print(f"❌ Error in worker count validation: {validation_error}")

# Global download monitor instance
download_monitor = WebUIDownloadMonitor()

def validate_and_heal_batch_states():
    """
    Periodic validation and healing of batch states to prevent permanent inconsistencies.
    This is the server-side equivalent of the frontend's worker count validation.
    """
    try:
        import time
        current_time = time.time()

        # Collect work to do outside the lock
        batches_needing_workers = []  # batch_ids that need _start_next_batch_of_downloads
        batches_needing_completion_check = []  # batch_ids that need _check_batch_completion_v2

        with tasks_lock:
            healed_batches = []
            batches_to_cleanup = []

            for batch_id, batch_data in list(download_batches.items()):
                active_count = batch_data.get('active_count', 0)
                queue = batch_data.get('queue', [])
                phase = batch_data.get('phase', 'unknown')

                # AUTO-CLEANUP: Remove completed batches after 5 minutes to prevent stale state
                if phase in ['complete', 'error', 'cancelled']:
                    # Check if batch has a completion timestamp
                    completion_time = batch_data.get('completion_time')
                    if not completion_time:
                        # Set completion time if not set
                        batch_data['completion_time'] = current_time
                    else:
                        # Check if batch has been complete for >5 minutes
                        time_since_completion = current_time - completion_time
                        if time_since_completion > 300:  # 5 minutes
                            print(f"🧹 [Auto-Cleanup] Removing stale completed batch {batch_id} (completed {time_since_completion:.0f}s ago)")
                            batches_to_cleanup.append(batch_id)
                            continue  # Skip other healing logic for this batch

                # Count actually active tasks
                actually_active = 0
                orphaned_tasks = []
                # Respect _on_download_completed dedup set — don't re-inflate active_count
                completed_task_ids = batch_data.get('_completed_task_ids', set())

                for task_id in queue:
                    if task_id in download_tasks:
                        task_status = download_tasks[task_id]['status']
                        if task_status in ['searching', 'downloading', 'queued', 'post_processing']:
                            if task_id not in completed_task_ids:
                                actually_active += 1
                        elif task_status in ['failed', 'completed', 'cancelled', 'not_found']:
                            orphaned_tasks.append(task_id)
                    else:
                        # Task in queue but not in download_tasks dict
                        orphaned_tasks.append(task_id)

                # Check for inconsistencies
                if active_count != actually_active:
                    print(f"🔧 [Batch Healing] {batch_id}: fixing active count {active_count} → {actually_active}")
                    batch_data['active_count'] = actually_active
                    healed_batches.append(batch_id)

                    # If we freed up slots, defer starting workers to outside the lock
                    if actually_active < batch_data.get('max_concurrent', 3):
                        queue_index = batch_data.get('queue_index', 0)
                        if queue_index < len(queue):
                            batches_needing_workers.append(batch_id)

                # Clean up orphaned tasks that are blocking progress
                if orphaned_tasks and phase == 'downloading':
                    print(f"🧹 [Batch Healing] Found {len(orphaned_tasks)} orphaned tasks in active batch {batch_id}")
                    batches_needing_completion_check.append(batch_id)

            # Cleanup stale batches inside the lock (safe - just dict mutations)
            for batch_id in batches_to_cleanup:
                task_ids_to_remove = download_batches[batch_id].get('queue', [])
                del download_batches[batch_id]
                # Clean up associated tasks
                for task_id in task_ids_to_remove:
                    if task_id in download_tasks:
                        del download_tasks[task_id]

            if batches_to_cleanup:
                print(f"🗑️ [Auto-Cleanup] Removed {len(batches_to_cleanup)} stale completed batches")

            if healed_batches:
                print(f"✅ [Batch Healing] Healed {len(healed_batches)} batches: {healed_batches}")

        # ---- All work below runs WITHOUT tasks_lock held ----

        # Start replacement workers for healed batches
        for batch_id in batches_needing_workers:
            try:
                print(f"🔄 [Batch Healing] Starting replacement workers for {batch_id}")
                _start_next_batch_of_downloads(batch_id)
            except Exception as e:
                print(f"❌ [Batch Healing] Error starting workers for {batch_id}: {e}")

        # Trigger completion checks for batches with orphaned tasks
        for batch_id in batches_needing_completion_check:
            try:
                print(f"🔄 [Batch Healing] Triggering completion check for batch with orphaned tasks")
                _check_batch_completion_v2(batch_id)
            except Exception as e:
                print(f"❌ [Batch Healing] Error checking completion for {batch_id}: {e}")
            
    except Exception as healing_error:
        print(f"❌ [Batch Healing] Error during validation: {healing_error}")

# Start periodic batch healing (every 30 seconds)
import threading
def start_batch_healing_timer():
    """Start periodic batch state validation and healing"""
    try:
        validate_and_heal_batch_states()
    except Exception as e:
        print(f"❌ [Batch Healing Timer] Error: {e}")
    finally:
        # Schedule next healing cycle
        threading.Timer(30.0, start_batch_healing_timer).start()

# Start the healing timer when the server starts
start_batch_healing_timer()

# Cleanup handler for Flask shutdown/reload
import atexit
import signal
import sys

def cleanup_monitor():
    """Clean up background monitor on shutdown"""
    if download_monitor.monitoring:
        print("🛑 Flask shutdown detected, stopping download monitor...")
        download_monitor.monitoring = False
        download_monitor.monitored_batches.clear()
        # Give the thread a moment to exit cleanly
        time.sleep(0.5)
        
    # Clean up batch locks to prevent memory leaks
    with tasks_lock:
        batch_locks.clear()
        print("🧹 Cleaned up batch locks")

# Global shutdown flag
IS_SHUTTING_DOWN = False

def signal_handler(signum, frame):
    """Handle SIGINT (Ctrl+C) and SIGTERM"""
    global IS_SHUTTING_DOWN
    print(f"🛑 Signal {signum} received, cleaning up...")
    IS_SHUTTING_DOWN = True
    cleanup_monitor()
    
    # Stop automation engine
    try:
        if automation_engine:
            print("🛑 Stopping automation engine...")
            automation_engine.stop()
    except Exception as e:
        print(f"⚠️ Error stopping automation engine: {e}")

    # Shutdown executor to prevent new tasks
    try:
        print("🛑 Shutting down missing_download_executor...")
        missing_download_executor.shutdown(wait=False, cancel_futures=True)
    except Exception as e:
        print(f"⚠️ Error shutting down executor: {e}")

    sys.exit(0)

# Register cleanup handlers
atexit.register(cleanup_monitor)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)

def _handle_failed_download(batch_id, task_id, task, task_status):
    """Handle failed download by triggering retry logic like GUI"""
    try:
        with tasks_lock:
            if task_id not in download_tasks:
                return
                
            retry_count = task.get('retry_count', 0)
            task['retry_count'] = retry_count + 1
            
            if task['retry_count'] > 2:  # Max 3 attempts total (matches GUI)
                # All retries exhausted, mark as permanently failed
                print(f"❌ Task {task_id} failed after 3 retry attempts")
                task_status['status'] = 'failed'
                task['status'] = 'failed'
                return
            
            # Show retrying status while we process retry
            task_status['status'] = 'pending'  # Will show as pending until retry kicks in
            print(f"🔄 Triggering retry {task['retry_count']}/3 for failed task {task_id}")
            
        # Trigger retry with next candidate (matches GUI retry_parallel_download_with_fallback)
        missing_download_executor.submit(download_monitor._retry_task_with_fallback, batch_id, task_id, task)
        
    except Exception as e:
        print(f"❌ Error handling failed download {task_id}: {e}")
        task_status['status'] = 'failed'
        task['status'] = 'failed'

def _update_task_status(task_id, new_status):
    """Helper to update task status and timestamp for timeout tracking"""
    with tasks_lock:
        if task_id in download_tasks:
            download_tasks[task_id]['status'] = new_status
            download_tasks[task_id]['status_change_time'] = time.time()

# --- Album Grouping State Management (Ported from GUI) ---
# Thread-safe album grouping for consistent naming across tracks
album_cache_lock = threading.Lock()
album_groups = {}  # album_key -> final_album_name
album_artists = {}  # album_key -> artist_name  
album_editions = {}  # album_key -> "standard" or "deluxe"
album_name_cache = {}  # album_key -> cached_final_name

# Thread-safe cache for MusicBrainz release MBID lookups.
# Prevents concurrent post-processing threads from getting different
# release IDs for the same album, which causes players like Navidrome
# to split one album into multiple entries.
_mb_release_cache = {}  # (album_lower, artist_lower) -> mbid_string_or_empty
_mb_release_cache_lock = threading.Lock()

def _prepare_stream_task(track_data):
    """
    Background streaming task that downloads track to Stream folder and updates global state.
    Enhanced version with robust error handling matching the GUI StreamingThread.
    """
    loop = None
    queue_start_time = None
    actively_downloading = False
    last_progress_sent = 0.0
    
    try:
        print(f"🎵 Starting stream preparation for: {track_data.get('filename')}")
        
        # Update state to loading
        with stream_lock:
            stream_state.update({
                "status": "loading",
                "progress": 0,
                "track_info": track_data,
                "file_path": None,
                "error_message": None
            })
        
        # Get paths
        download_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
        project_root = os.path.dirname(os.path.abspath(__file__))
        stream_folder = os.path.join(project_root, 'Stream')
        
        # Ensure Stream directory exists
        os.makedirs(stream_folder, exist_ok=True)
        
        # Clear any existing files in Stream folder (only one file at a time)
        for existing_file in glob.glob(os.path.join(stream_folder, '*')):
            try:
                if os.path.isfile(existing_file):
                    os.remove(existing_file)
                elif os.path.isdir(existing_file):
                    shutil.rmtree(existing_file)
                print(f"🗑️ Cleared old stream file: {existing_file}")
            except Exception as e:
                print(f"⚠️ Could not remove existing stream file: {e}")
        
        # Start the download using the same mechanism as regular downloads
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        
        try:
            download_result = loop.run_until_complete(soulseek_client.download(
                track_data.get('username'),
                track_data.get('filename'),
                track_data.get('size', 0)
            ))
            
            if not download_result:
                with stream_lock:
                    stream_state.update({
                        "status": "error",
                        "error_message": "Failed to initiate download - uploader may be offline"
                    })
                return
            
            print(f"✓ Download initiated for streaming")
            
            # Enhanced monitoring with queue timeout detection (matching GUI)
            max_wait_time = 60  # Increased timeout
            poll_interval = 1.5  # More frequent polling
            queue_timeout = 15   # Queue timeout like GUI
            wait_count = 0
            
            while wait_count * poll_interval < max_wait_time:
                wait_count += 1
                
                # Check download progress via orchestrator (works for Soulseek and YouTube)
                api_progress = None
                download_state = None
                download_status = None

                try:
                    # Use orchestrator's get_all_downloads() which works for both sources
                    all_downloads = loop.run_until_complete(soulseek_client.get_all_downloads())
                    download_status = _find_streaming_download_in_all_downloads(all_downloads, track_data)
                    
                    if download_status:
                        api_progress = download_status.get('percentComplete', 0)
                        download_state = download_status.get('state', '').lower()
                        original_state = download_status.get('state', '')
                        
                        print(f"API Download - State: {original_state}, Progress: {api_progress:.1f}%")
                        
                        # Track queue state timing (matching GUI logic)
                        is_queued = ('queued' in download_state or 'initializing' in download_state)
                        is_downloading = ('inprogress' in download_state or 'transferring' in download_state)
                        # Verify bytes match before trusting state/progress
                        _stream_expected = download_status.get('size', 0)
                        _stream_transferred = download_status.get('bytesTransferred', 0)
                        _bytes_ok = _stream_expected <= 0 or _stream_transferred >= _stream_expected
                        is_completed = ('succeeded' in download_state or api_progress >= 100) and _bytes_ok
                        
                        # Handle queue state timing
                        if is_queued and queue_start_time is None:
                            queue_start_time = time.time()
                            print(f"📋 Download entered queue state: {original_state}")
                            with stream_lock:
                                stream_state["status"] = "queued"
                        elif is_downloading and not actively_downloading:
                            actively_downloading = True
                            queue_start_time = None  # Reset queue timer
                            print(f"🚀 Download started actively downloading: {original_state}")
                            with stream_lock:
                                stream_state["status"] = "loading"
                        
                        # Check for queue timeout (matching GUI)
                        if is_queued and queue_start_time:
                            queue_elapsed = time.time() - queue_start_time
                            if queue_elapsed > queue_timeout:
                                print(f"⏰ Queue timeout after {queue_elapsed:.1f}s - download stuck in queue")
                                with stream_lock:
                                    stream_state.update({
                                        "status": "error",
                                        "error_message": "Queue timeout - uploader not responding. Try another source."
                                    })
                                return
                        
                        # Update progress
                        with stream_lock:
                            if api_progress != last_progress_sent:
                                stream_state["progress"] = api_progress
                                last_progress_sent = api_progress
                        
                        # Check if download is complete
                        if is_completed:
                            print(f"✓ Download completed via API status: {original_state}")

                            # Wait for file to stabilise on disk before moving
                            found_file = _find_downloaded_file(download_path, track_data)
                            if found_file:
                                _prev_sz = -1
                                for _sc in range(4):
                                    try:
                                        _cur_sz = os.path.getsize(found_file)
                                    except OSError:
                                        _cur_sz = -1
                                    if _cur_sz == _prev_sz and _cur_sz > 0:
                                        break
                                    _prev_sz = _cur_sz
                                    time.sleep(1.5)

                            # Re-find in case it wasn't found on first try
                            if not found_file:
                                found_file = _find_downloaded_file(download_path, track_data)
                            
                            # Retry file search a few times (matching GUI logic)
                            retry_attempts = 5
                            for attempt in range(retry_attempts):
                                if found_file:
                                    break
                                print(f"File not found yet, attempt {attempt + 1}/{retry_attempts}")
                                time.sleep(1)
                                found_file = _find_downloaded_file(download_path, track_data)
                            
                            if found_file:
                                print(f"✓ Found downloaded file: {found_file}")
                                
                                # Move file to Stream folder
                                original_filename = extract_filename(found_file)
                                stream_path = os.path.join(stream_folder, original_filename)
                                
                                try:
                                    shutil.move(found_file, stream_path)
                                    print(f"✓ Moved file to stream folder: {stream_path}")
                                    
                                    # Clean up empty directories (matching GUI)
                                    _cleanup_empty_directories(download_path, found_file)
                                    
                                    # Update state to ready
                                    with stream_lock:
                                        stream_state.update({
                                            "status": "ready",
                                            "progress": 100,
                                            "file_path": stream_path
                                        })
                                    
                                    # Clean up download from slskd API
                                    try:
                                        download_id = download_status.get('id', '')
                                        if download_id and track_data.get('username'):
                                            success = loop.run_until_complete(
                                                soulseek_client.signal_download_completion(
                                                    download_id, track_data.get('username'), remove=True)
                                            )
                                            if success:
                                                print(f"✓ Cleaned up download {download_id} from API")
                                    except Exception as e:
                                        print(f"⚠️ Error cleaning up download: {e}")
                                    
                                    print(f"✅ Stream file ready for playback: {stream_path}")
                                    return  # Success!
                                    
                                except Exception as e:
                                    print(f"❌ Error moving file to stream folder: {e}")
                                    with stream_lock:
                                        stream_state.update({
                                            "status": "error",
                                            "error_message": f"Failed to prepare stream file: {e}"
                                        })
                                    return
                            else:
                                print("❌ Could not find downloaded file after completion")
                                with stream_lock:
                                    stream_state.update({
                                        "status": "error",
                                        "error_message": "Download completed but file not found"
                                    })
                                return
                    else:
                        # No transfer found in API - may still be initializing
                        print(f"No transfer found in API yet... (elapsed: {wait_count * poll_interval}s)")
                        
                except Exception as e:
                    print(f"⚠️ Error checking download progress: {e}")
                    # Continue to next iteration if API call fails
                
                # Wait before next poll
                time.sleep(poll_interval)
            
            # If we get here, download timed out
            print(f"❌ Download timed out after {max_wait_time}s")
            with stream_lock:
                stream_state.update({
                    "status": "error", 
                    "error_message": "Download timed out - try a different source"
                })
                
        except asyncio.CancelledError:
            print("🛑 Stream task cancelled")
            with stream_lock:
                stream_state.update({
                    "status": "stopped",
                    "error_message": None
                })
        finally:
            if loop:
                try:
                    # Clean up any pending tasks
                    pending = asyncio.all_tasks(loop)
                    if pending:
                        loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
                    loop.close()
                except Exception as e:
                    print(f"⚠️ Error cleaning up streaming event loop: {e}")
            
    except Exception as e:
        print(f"❌ Stream preparation failed: {e}")
        with stream_lock:
            stream_state.update({
                "status": "error",
                "error_message": f"Streaming error: {str(e)}"
            })

def _find_streaming_download_in_all_downloads(all_downloads, track_data):
    """
    Find streaming download in DownloadStatus list (works for Soulseek, YouTube, and Tidal).
    Replaces the old _find_streaming_download_in_transfers function.
    """
    try:
        if not all_downloads:
            return None

        # Look for our specific file by filename and username
        target_filename = extract_filename(track_data.get('filename', ''))
        target_username = track_data.get('username', '')

        for download in all_downloads:
            download_filename = extract_filename(download.filename)
            download_username = download.username

            if (download_filename == target_filename and
                download_username == target_username):
                # Convert DownloadStatus to dict format expected by caller
                return {
                    'percentComplete': download.progress,
                    'state': download.state,
                    'size': download.size,
                    'bytesTransferred': download.transferred,
                    'averageSpeed': download.speed,
                }

        return None
    except Exception as e:
        print(f"Error finding streaming download: {e}")
        return None

def _find_downloaded_file(download_path, track_data):
    """Find the downloaded audio file in the downloads directory tree (works for Soulseek, YouTube, and Tidal)"""
    # Ensure path is accessible in Docker (handles E:/ -> /host/mnt/e/)
    download_path = docker_resolve_path(download_path)
    
    audio_extensions = {'.mp3', '.flac', '.ogg', '.aac', '.wma', '.wav', '.m4a'}
    target_filename = extract_filename(track_data.get('filename', ''))

    # YOUTUBE/TIDAL SUPPORT: Handle encoded filename format "id||title"
    # The file on disk will be "title.ext", not "id||title"
    is_youtube = track_data.get('username') == 'youtube'
    is_tidal = track_data.get('username') == 'tidal'
    is_streaming_source = is_youtube or is_tidal
    target_filename_youtube = None
    if is_streaming_source and '||' in target_filename:
        _, title = target_filename.split('||', 1)
        if is_tidal:
            # Tidal files can be flac or m4a — match any audio extension
            safe_title = re.sub(r'[<>:"/\\|?*]', '_', title)
            target_filename_youtube = safe_title  # Extension-less for flexible matching
            print(f"🎵 [Tidal Stream] Looking for file starting with: {target_filename_youtube}")
        else:
            # yt-dlp will create "Title.mp3" from "Title"
            target_filename_youtube = f"{title}.mp3"
            print(f"🎵 [YouTube Stream] Looking for file: {target_filename_youtube}")
    elif is_streaming_source:
        # Fallback: if streaming source but no encoded format, use as-is
        target_filename_youtube = target_filename
        print(f"🎵 [Stream] Using direct filename: {target_filename_youtube}")

    try:
        # Walk through the downloads directory to find the file
        best_match = None
        best_similarity = 0.0

        for root, dirs, files in os.walk(download_path):
            for file in files:
                # Skip non-audio files
                if os.path.splitext(file)[1].lower() not in audio_extensions:
                    continue

                file_path = os.path.join(root, file)

                # Skip empty files
                try:
                    if os.path.getsize(file_path) < 1024:  # At least 1KB
                        continue
                except:
                    continue

                # Check if this is our target file
                if is_streaming_source and target_filename_youtube:
                    # For YouTube/Tidal, use fuzzy matching (case-insensitive, flexible)
                    from difflib import SequenceMatcher
                    # For Tidal, compare without extension (file could be .flac or .m4a)
                    compare_target = target_filename_youtube.lower()
                    compare_file = file.lower()
                    if is_tidal:
                        compare_file = os.path.splitext(compare_file)[0]
                    similarity = SequenceMatcher(None, compare_file, compare_target).ratio()

                    source_label = 'Tidal' if is_tidal else 'YouTube'
                    print(f"🔍 [{source_label} Stream] Comparing: '{file}' vs '{target_filename_youtube}' = {similarity:.2f}")

                    # Keep track of best match
                    if similarity > best_similarity:
                        best_similarity = similarity
                        best_match = file_path

                        # If we have a very good match (95%+), use it immediately
                        if similarity >= 0.95:
                            print(f"✅ Found excellent match for streaming file: {file_path}")
                            return file_path
                else:
                    # For Soulseek, exact match
                    if file == target_filename:
                        print(f"✅ Found streaming file: {file_path}")
                        return file_path

        # For YouTube/Tidal, if we found a good enough match (80%+), use it
        if is_streaming_source and best_match and best_similarity >= 0.80:
            source_label = 'Tidal' if is_tidal else 'YouTube'
            print(f"✅ Found good match ({best_similarity:.2f}) for {source_label} streaming file: {best_match}")
            return best_match

        print(f"❌ Could not find downloaded file: {target_filename}")
        if is_streaming_source:
            print(f"   Looking for: {target_filename_youtube}")
            print(f"   Best similarity: {best_similarity:.2f}")
        return None

    except Exception as e:
        print(f"Error searching for downloaded file: {e}")
        return None

# --- Refactored Logic from GUI Threads ---
# This logic is extracted from your QThread classes to be used directly by Flask.

def run_service_test(service, test_config):
    """
    Performs the actual connection test for a given service.
    This logic is adapted from your ServiceTestThread.
    It temporarily modifies the config, runs the test, then restores the config.
    """
    original_config = {}
    try:
        # 1. Save original config for the specific service
        original_config = config_manager.get(service, {})

        # 2. Temporarily set the new config for the test (with Docker URL resolution)
        for key, value in test_config.items():
            # Apply Docker URL resolution for URL/URI fields
            if isinstance(value, str) and ('url' in key.lower() or 'uri' in key.lower()):
                value = docker_resolve_url(value)
            config_manager.set(f"{service}.{key}", value)

        # 3. Run the test with the temporary config
        if service == "spotify":
            temp_client = SpotifyClient()
            
            # Check if Spotify credentials are configured
            spotify_config = config_manager.get('spotify', {})
            spotify_configured = bool(spotify_config.get('client_id') and spotify_config.get('client_secret'))
            
            if temp_client.is_authenticated():
                 # Determine which source is active
                 if temp_client.is_spotify_authenticated():
                     return True, "Spotify connection successful!"
                 else:
                     # Using iTunes fallback
                     if spotify_configured:
                         return True, "Apple Music connection successful! (Spotify configured but not authenticated)"
                     else:
                         return True, "Apple Music connection successful! (Spotify not configured)"
            else:
                 return False, "Music service authentication failed. Check credentials and complete OAuth flow in browser if prompted."
        elif service == "tidal":
            temp_client = TidalClient()
            if temp_client.is_authenticated():
                user_info = temp_client.get_user_info()
                username = user_info.get('display_name', 'Tidal User') if user_info else 'Tidal User'
                return True, f"Tidal connection successful! Connected as: {username}"
            else:
                return False, "Tidal authentication failed. Please use the 'Authenticate' button and complete the flow in your browser."
        elif service == "plex":
            temp_client = PlexClient()
            if temp_client.is_connected():
                return True, f"Successfully connected to Plex server: {temp_client.server.friendlyName}"
            else:
                return False, "Could not connect to Plex. Check URL and Token."
        elif service == "jellyfin":
            temp_client = JellyfinClient()
            if temp_client.is_connected():
                # FIX: Check if server_info exists before accessing it.
                server_name = "Unknown Server"
                if hasattr(temp_client, 'server_info') and temp_client.server_info:
                    server_name = temp_client.server_info.get('ServerName', 'Unknown Server')
                return True, f"Successfully connected to Jellyfin server: {server_name}"
            else:
                return False, "Could not connect to Jellyfin. Check URL and API Key."
        elif service == "navidrome":
            # Test Navidrome connection using Subsonic API
            base_url = test_config.get('base_url', '')
            username = test_config.get('username', '')
            password = test_config.get('password', '')

            if not all([base_url, username, password]):
                return False, "Missing Navidrome URL, username, or password."

            try:
                import hashlib
                import random
                import string

                # Generate salt and token for Subsonic API authentication
                salt = ''.join(random.choices(string.ascii_letters + string.digits, k=6))
                token = hashlib.md5((password + salt).encode()).hexdigest()

                # Test ping endpoint
                url = f"{base_url.rstrip('/')}/rest/ping"
                response = requests.get(url, params={
                    'u': username,
                    't': token,
                    's': salt,
                    'v': '1.16.1',
                    'c': 'soulsync',
                    'f': 'json'
                }, timeout=5)

                if response.status_code == 200:
                    data = response.json()
                    if data.get('subsonic-response', {}).get('status') == 'ok':
                        server_version = data.get('subsonic-response', {}).get('version', 'Unknown')
                        return True, f"Successfully connected to Navidrome server (v{server_version})"
                    else:
                        error = data.get('subsonic-response', {}).get('error', {})
                        return False, f"Navidrome authentication failed: {error.get('message', 'Unknown error')}"
                else:
                    return False, f"Could not connect to Navidrome server (HTTP {response.status_code})"

            except Exception as e:
                return False, f"Navidrome connection error: {str(e)}"
        elif service == "soulseek":
            # Test the orchestrator's configured download source (not just Soulseek)
            download_mode = config_manager.get('download_source.mode', 'soulseek')

            if run_async(soulseek_client.check_connection()):
                # Success message based on active mode
                mode_messages = {
                    'soulseek': "Successfully connected to Soulseek network via slskd.",
                    'youtube': "YouTube download source ready.",
                    'tidal': "Tidal download source ready.",
                    'hybrid': "Download sources ready (Hybrid mode)."
                }
                message = mode_messages.get(download_mode, "Download source connected.")
                return True, message
            else:
                # Failure message based on active mode
                mode_errors = {
                    'soulseek': "slskd is not connected to the Soulseek network. Check slskd status and credentials.",
                    'youtube': "YouTube download source not available.",
                    'tidal': "Tidal download source not available. Check authentication.",
                    'hybrid': "Could not connect to download sources. Check configuration."
                }
                error = mode_errors.get(download_mode, "Download source connection failed.")
                return False, error
        elif service == "listenbrainz":
            token = test_config.get('token', '')

            if not token:
                return False, "Missing ListenBrainz user token."

            try:
                # Test ListenBrainz API by validating the token
                url = "https://api.listenbrainz.org/1/validate-token"
                headers = {
                    'Authorization': f'Token {token}'
                }
                response = requests.get(url, headers=headers, timeout=5)

                if response.status_code == 200:
                    data = response.json()
                    if data.get('valid'):
                        username = data.get('user_name', 'Unknown')
                        return True, f"Successfully connected to ListenBrainz! Connected as: {username}"
                    else:
                        return False, "Invalid ListenBrainz token."
                elif response.status_code == 401:
                    return False, "Invalid ListenBrainz token (unauthorized)."
                else:
                    return False, f"Could not connect to ListenBrainz (HTTP {response.status_code})"

            except Exception as e:
                return False, f"ListenBrainz connection error: {str(e)}"
        elif service == "acoustid":
            api_key = test_config.get('api_key', '')

            if not api_key:
                return False, "Missing AcoustID API key."

            try:
                from core.acoustid_client import AcoustIDClient, CHROMAPRINT_AVAILABLE, ACOUSTID_AVAILABLE, FPCALC_PATH

                if not ACOUSTID_AVAILABLE:
                    return False, "pyacoustid library not installed. Run: pip install pyacoustid"

                client = AcoustIDClient()

                # Override the cached API key with the test config key
                client._api_key = api_key

                # Check chromaprint/fpcalc availability
                if CHROMAPRINT_AVAILABLE and FPCALC_PATH:
                    fingerprint_status = f"fpcalc ready: {FPCALC_PATH}"
                elif CHROMAPRINT_AVAILABLE:
                    fingerprint_status = "Fingerprint backend available"
                else:
                    fingerprint_status = "fpcalc not found (will auto-download on first use)"

                # Validate API key with test request
                success, message = client.test_api_key()
                if success:
                    return True, f"AcoustID API key is valid! {fingerprint_status}"
                else:
                    return False, f"{message}. {fingerprint_status}"
            except Exception as e:
                return False, f"AcoustID test error: {str(e)}"
        return False, "Unknown service."
    except AttributeError as e:
        # This specifically catches the error you reported for Jellyfin
        if "'JellyfinClient' object has no attribute 'server_info'" in str(e):
            return False, "Connection failed. Please check your Jellyfin URL and API Key."
        else:
            return False, f"An unexpected error occurred: {e}"
    except Exception as e:
        import traceback
        traceback.print_exc()
        return False, str(e)
    finally:
        # 4. CRITICAL: Restore the original config
        if original_config:
            for key, value in original_config.items():
                config_manager.set(f"{service}.{key}", value)
            print(f"✅ Restored original config for '{service}' after test.")


def run_detection(server_type):
    """
    Performs comprehensive network detection for a given server type (plex, jellyfin, slskd).
    This implements the same scanning logic as the GUI's detection threads.
    """
    print(f"Running comprehensive detection for {server_type}...")
    
    def get_network_info():
        """Get comprehensive network information with subnet detection"""
        try:
            # Get local IP using socket method
            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            s.connect(("8.8.8.8", 80))
            local_ip = s.getsockname()[0]
            s.close()
            
            # Try to get actual subnet mask
            try:
                if platform.system() == "Windows":
                    # Windows: Use netsh to get subnet info
                    result = subprocess.run(['netsh', 'interface', 'ip', 'show', 'config'], 
                                          capture_output=True, text=True, timeout=3)
                    # Parse output for subnet mask (simplified)
                    subnet_mask = "255.255.255.0"  # Default fallback
                else:
                    # Linux/Mac: Try to parse network interfaces
                    result = subprocess.run(['ip', 'route', 'show'], 
                                          capture_output=True, text=True, timeout=3)
                    subnet_mask = "255.255.255.0"  # Default fallback
            except:
                subnet_mask = "255.255.255.0"  # Default /24
            
            # Calculate network range
            network = ipaddress.IPv4Network(f"{local_ip}/{subnet_mask}", strict=False)
            return str(network.network_address), str(network.netmask), local_ip, network
            
        except Exception as e:
            # Fallback to original method
            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            s.connect(("8.8.8.8", 80))
            local_ip = s.getsockname()[0]
            s.close()
            
            # Default to /24 network
            network = ipaddress.IPv4Network(f"{local_ip}/24", strict=False)
            return str(network.network_address), "255.255.255.0", local_ip, network

    def test_plex_server(ip, port=32400):
        """Test if a Plex server is running at the given IP and port"""
        try:
            url = f"http://{ip}:{port}/web/index.html"
            response = requests.get(url, timeout=2, allow_redirects=True)
            
            # Check for Plex-specific indicators
            if response.status_code == 200:
                # Check if it's actually Plex
                if 'plex' in response.text.lower() or 'X-Plex' in str(response.headers):
                    return f"http://{ip}:{port}"
                    
                # Also try the API endpoint
                api_url = f"http://{ip}:{port}/identity"
                api_response = requests.get(api_url, timeout=1)
                if api_response.status_code == 200 and 'MediaContainer' in api_response.text:
                    return f"http://{ip}:{port}"
                    
        except:
            pass
        return None

    def test_jellyfin_server(ip, port=8096):
        """Test if a Jellyfin server is running at the given IP and port"""
        try:
            # Try the system info endpoint first
            url = f"http://{ip}:{port}/System/Info"
            response = requests.get(url, timeout=2, allow_redirects=True)
            
            if response.status_code == 200:
                # Check if response contains Jellyfin-specific content
                if 'jellyfin' in response.text.lower() or 'ServerName' in response.text:
                    return f"http://{ip}:{port}"
            
            # Also try the web interface
            web_url = f"http://{ip}:{port}/web/index.html"
            web_response = requests.get(web_url, timeout=1)
            if web_response.status_code == 200 and 'jellyfin' in web_response.text.lower():
                return f"http://{ip}:{port}"
                
        except:
            pass
        return None

    def test_slskd_server(ip, port=5030):
        """Test if a slskd server is running at the given IP and port"""
        try:
            # slskd specific API endpoint
            url = f"http://{ip}:{port}/api/v0/session"
            response = requests.get(url, timeout=2)
            
            # slskd returns 401 when not authenticated, which is still a valid response
            if response.status_code in [200, 401]:
                return f"http://{ip}:{port}"

        except:
            pass
        return None

    def test_navidrome_server(ip, port=4533):
        """Test if a Navidrome server is running at the given IP and port"""
        try:
            # Try Navidrome's ping endpoint (part of Subsonic API)
            url = f"http://{ip}:{port}/rest/ping"
            response = requests.get(url, timeout=2, params={
                'u': 'test',  # Dummy username for ping test
                'v': '1.16.1',  # API version
                'c': 'soulsync',  # Client name
                'f': 'json'  # Response format
            })

            # Navidrome should respond even with invalid credentials for ping
            if response.status_code in [200, 401, 403]:
                try:
                    data = response.json()
                    # Check for Subsonic/Navidrome API response structure
                    if 'subsonic-response' in data:
                        return f"http://{ip}:{port}"
                except:
                    pass

            # Also try the web interface
            web_url = f"http://{ip}:{port}/"
            web_response = requests.get(web_url, timeout=2)
            if web_response.status_code == 200 and 'navidrome' in web_response.text.lower():
                return f"http://{ip}:{port}"

        except:
            pass
        return None

    try:
        network_addr, netmask, local_ip, network = get_network_info()
        
        # Select the appropriate test function
        test_functions = {
            'plex': test_plex_server,
            'jellyfin': test_jellyfin_server,
            'navidrome': test_navidrome_server,
            'slskd': test_slskd_server
        }
        
        test_func = test_functions.get(server_type)
        if not test_func:
            return None
        
        # Priority 1: Test localhost first
        print(f"Testing localhost for {server_type}...")
        localhost_result = test_func("localhost")
        if localhost_result:
            print(f"Found {server_type} at localhost!")
            return localhost_result
        
        # Priority 1.5: In Docker, try Docker host IP
        import os
        if os.path.exists('/.dockerenv'):
            print(f"Docker detected, testing Docker host for {server_type}...")
            try:
                # Try host.docker.internal (Windows/Mac)
                host_result = test_func("host.docker.internal")
                if host_result:
                    print(f"Found {server_type} at Docker host!")
                    return host_result.replace("host.docker.internal", "localhost")  # Convert back to localhost for config
                
                # Try Docker bridge gateway (Linux)
                gateway_result = test_func("172.17.0.1")
                if gateway_result:
                    print(f"Found {server_type} at Docker gateway!")
                    return gateway_result.replace("172.17.0.1", "localhost")  # Convert back to localhost for config
            except Exception as e:
                print(f"Docker host detection failed: {e}")
        
        # Priority 2: Test local IP
        print(f"Testing local IP {local_ip} for {server_type}...")
        local_result = test_func(local_ip)
        if local_result:
            print(f"Found {server_type} at {local_ip}!")
            return local_result
        
        # Priority 3: Test common IPs (router gateway, etc.)
        common_ips = [
            local_ip.rsplit('.', 1)[0] + '.1',  # Typical gateway
            local_ip.rsplit('.', 1)[0] + '.2',  # Alternative gateway
            local_ip.rsplit('.', 1)[0] + '.100', # Common static IP
        ]
        
        print(f"Testing common IPs for {server_type}...")
        for ip in common_ips:
            print(f"  Checking {ip}...")
            result = test_func(ip)
            if result:
                print(f"Found {server_type} at {ip}!")
                return result
        
        # Priority 4: Scan the network range (limited to reasonable size)
        network_hosts = list(network.hosts())
        if len(network_hosts) > 50:
            # Limit scan to reasonable size for performance
            step = max(1, len(network_hosts) // 50)
            network_hosts = network_hosts[::step]
        
        print(f"Scanning network range for {server_type} ({len(network_hosts)} hosts)...")
        
        # Use ThreadPoolExecutor for concurrent scanning (limited for web context)
        with ThreadPoolExecutor(max_workers=5) as executor:
            # Submit all tasks
            future_to_ip = {executor.submit(test_func, str(ip)): str(ip) 
                           for ip in network_hosts}
            
            try:
                for future in as_completed(future_to_ip):
                    ip = future_to_ip[future]
                    try:
                        result = future.result()
                        if result:
                            print(f"Found {server_type} at {ip}!")
                            # Cancel all pending futures before returning
                            for f in future_to_ip:
                                if not f.done():
                                    f.cancel()
                            return result
                    except Exception as e:
                        print(f"Error testing {ip}: {e}")
                        continue
            except Exception as e:
                print(f"Error in concurrent scanning: {e}")
        
        print(f"No {server_type} server found on network")
        return None
        
    except Exception as e:
        print(f"Error during {server_type} detection: {e}")
        return None

# --- Web UI Routes ---

@app.route('/')
def index():
    return render_template('index.html')

# --- API Endpoints ---

# Status check caching to reduce unnecessary API calls
_status_cache = {
    'spotify': {'connected': False, 'response_time': 0, 'source': 'itunes'},
    'media_server': {'connected': False, 'response_time': 0, 'type': None},
    'soulseek': {'connected': False, 'response_time': 0}
}
_status_cache_timestamps = {
    'spotify': 0,
    'media_server': 0,
    'soulseek': 0
}
STATUS_CACHE_TTL = 120  # Cache for 2 minutes (reduces API calls while staying fresh)

@app.route('/status')
def get_status():
    if not all([spotify_client, plex_client, jellyfin_client, soulseek_client, config_manager]):
        return jsonify({"error": "Core services not initialized."}), 500
    try:
        import time

        current_time = time.time()
        active_server = config_manager.get_active_media_server()

        # Test Spotify - with caching to avoid excessive API calls
        if current_time - _status_cache_timestamps['spotify'] > STATUS_CACHE_TTL:
            spotify_start = time.time()
            # Single auth check — is_spotify_authenticated() is cached internally (60s TTL)
            spotify_connected = spotify_client.is_spotify_authenticated()
            spotify_response_time = (time.time() - spotify_start) * 1000

            music_source = 'spotify' if spotify_connected else 'itunes'

            _status_cache['spotify'] = {
                'connected': True,  # Always true — iTunes fallback is always available
                'response_time': round(spotify_response_time, 1),
                'source': music_source
            }
            _status_cache_timestamps['spotify'] = current_time
        # else: use cached value

        # Test media server - use EXISTING instances (they have internal caching)
        # Media server clients already cache connection checks internally
        if current_time - _status_cache_timestamps['media_server'] > STATUS_CACHE_TTL:
            media_server_start = time.time()
            media_server_status = False
            if active_server == "plex":
                # Use existing instance - has 30s internal connection cache
                media_server_status = plex_client.is_connected()
            elif active_server == "jellyfin":
                # Use existing instance - has internal connection caching
                media_server_status = jellyfin_client.is_connected()
            elif active_server == "navidrome":
                # Use existing instance
                media_server_status = navidrome_client.is_connected()
            media_server_response_time = (time.time() - media_server_start) * 1000
            _status_cache['media_server'] = {
                'connected': media_server_status,
                'response_time': round(media_server_response_time, 1),
                'type': active_server
            }
            _status_cache_timestamps['media_server'] = current_time
        # else: use cached value

        # Test Soulseek - with caching like other services
        if current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL:
            soulseek_start = time.time()
            try:
                soulseek_status = run_async(soulseek_client.check_connection())
            except Exception:
                soulseek_status = False
            soulseek_response_time = (time.time() - soulseek_start) * 1000
            _status_cache['soulseek'] = {
                'connected': soulseek_status,
                'response_time': round(soulseek_response_time, 1)
            }
            _status_cache_timestamps['soulseek'] = current_time

        # Include download source mode so frontend can update labels
        download_mode = config_manager.get('download_source.mode', 'soulseek')
        _status_cache['soulseek']['source'] = download_mode

        status_data = {
            'spotify': _status_cache['spotify'],
            'media_server': _status_cache['media_server'],
            'soulseek': _status_cache['soulseek'],
            'active_media_server': active_server
        }
        return jsonify(status_data)
    except Exception as e:
        return jsonify({'error': str(e)}), 500

@app.route('/api/fix-navidrome-urls', methods=['POST'])
def fix_navidrome_urls():
    """Fix Navidrome artist image URLs to use correct Subsonic format"""
    try:
        db = get_database()
        with db._get_connection() as conn:
            cursor = conn.cursor()

            # Get all Navidrome artists with old URL format
            cursor.execute('SELECT id, name, thumb_url FROM artists WHERE server_source = "navidrome" AND thumb_url LIKE "/api/artist/%"')
            artists = cursor.fetchall()

            if not artists:
                return jsonify({"status": "success", "message": "No URLs needed fixing", "updated": 0})

            # Update URLs to new Subsonic format
            import re
            updated = 0
            examples = []

            for artist_id, name, old_url in artists:
                # Extract artist ID from old URL: /api/artist/ARTIST_ID/image
                match = re.search(r'/api/artist/([^/]+)/image', old_url)
                if match:
                    artist_spotify_id = match.group(1)
                    new_url = f'/rest/getCoverArt?id={artist_spotify_id}'

                    cursor.execute('UPDATE artists SET thumb_url = ? WHERE id = ? AND server_source = "navidrome"', (new_url, artist_id))
                    updated += 1

                    if len(examples) < 3:  # Show first 3 as examples
                        examples.append(f'{name}: {old_url} -> {new_url}')

            conn.commit()

            return jsonify({
                "status": "success",
                "message": f"Updated {updated} Navidrome artist URLs to Subsonic format",
                "updated": updated,
                "examples": examples
            })

    except Exception as e:
        return jsonify({"status": "error", "message": str(e)}), 500

@app.route('/api/save-playlist-m3u', methods=['POST'])
def save_playlist_m3u():
    """Save M3U playlist file to the relevant download folder"""
    try:
        data = request.get_json()
        if not data:
            return jsonify({"status": "error", "message": "No data provided"}), 400

        playlist_name = data.get('playlist_name', 'Playlist')
        m3u_content = data.get('m3u_content', '')
        context_type = data.get('context_type', 'playlist')
        artist_name = data.get('artist_name', '')
        album_name = data.get('album_name', '')
        year = data.get('year', '')
        force = data.get('force', False)

        # Check if M3U export is enabled (unless force=True from manual Export button)
        if not force and not config_manager.get('m3u_export.enabled', False):
            return jsonify({"status": "success", "message": "M3U export disabled in settings", "skipped": True})

        if not m3u_content:
            return jsonify({"status": "error", "message": "No M3U content provided"}), 400

        # Compute target folder using the template system
        transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer'))
        m3u_folder = _compute_m3u_folder(transfer_dir, context_type, playlist_name, artist_name, album_name, year)
        os.makedirs(m3u_folder, exist_ok=True)

        # Build M3U filename from playlist or album name
        if context_type == 'album' and artist_name and album_name:
            safe_filename = _sanitize_filename(f"{artist_name} - {album_name}")
        else:
            safe_filename = _sanitize_filename(playlist_name)
        m3u_filename = f"{safe_filename}.m3u"
        m3u_path = os.path.join(m3u_folder, m3u_filename)

        # Write M3U file (overwrite if exists)
        with open(m3u_path, 'w', encoding='utf-8') as f:
            f.write(m3u_content)

        logger.info(f"✅ Saved M3U file: {m3u_path}")
        return jsonify({
            "status": "success",
            "message": f"M3U file saved: {m3u_filename}",
            "path": m3u_path
        })

    except Exception as e:
        logger.error(f"❌ Error saving M3U file: {e}")
        return jsonify({"status": "error", "message": str(e)}), 500

def _build_system_stats():
    """Build system statistics dict — shared by HTTP handler and WebSocket emitter."""
    import psutil
    import time
    from datetime import timedelta

    # Calculate uptime
    start_time = getattr(app, 'start_time', time.time())
    uptime_seconds = time.time() - start_time
    uptime = str(timedelta(seconds=int(uptime_seconds)))

    # Get memory usage
    memory = psutil.virtual_memory()
    memory_usage = f"{memory.percent}%"

    # Count active downloads from download_batches (batches that are currently downloading)
    active_downloads = len([batch_id for batch_id, batch_data in download_batches.items()
                           if batch_data.get('phase') == 'downloading'])

    # Count finished downloads (completed this session) - use session counter like dashboard.py
    with session_stats_lock:
        finished_downloads = session_completed_downloads

    # Calculate total download speed from active soulseek transfers
    total_download_speed = 0.0
    try:
        transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
        if transfers_data:
            for user_data in transfers_data:
                if 'directories' in user_data:
                    for directory in user_data['directories']:
                        if 'files' in directory:
                            for file_info in directory['files']:
                                state = file_info.get('state', '').lower()
                                # Only count actively downloading files
                                if 'inprogress' in state or 'downloading' in state or 'transferring' in state:
                                    speed = file_info.get('averageSpeed', 0)
                                    if isinstance(speed, (int, float)) and speed > 0:
                                        total_download_speed += float(speed)
    except Exception as e:
        print(f"Warning: Could not fetch download speeds: {e}")

    # Convert bytes/sec to KB/s and format
    if total_download_speed > 0:
        speed_kb_s = total_download_speed / 1024
        if speed_kb_s >= 1024:
            speed_mb_s = speed_kb_s / 1024
            download_speed_str = f"{speed_mb_s:.1f} MB/s"
        else:
            download_speed_str = f"{speed_kb_s:.1f} KB/s"
    else:
        download_speed_str = "0 KB/s"

    # Count active syncs (playlists currently syncing)
    active_syncs = 0
    # Count Spotify playlist syncs
    for playlist_id, sync_state in sync_states.items():
        if sync_state.get('status') == 'syncing':
            active_syncs += 1
    # Count YouTube playlist syncs
    for url_hash, state in youtube_playlist_states.items():
        if state.get('phase') == 'syncing':
            active_syncs += 1
    # Count Tidal playlist syncs
    for playlist_id, state in tidal_discovery_states.items():
        if state.get('phase') == 'syncing':
            active_syncs += 1

    return {
        'active_downloads': active_downloads,
        'finished_downloads': finished_downloads,
        'download_speed': download_speed_str,
        'active_syncs': active_syncs,
        'uptime': uptime,
        'memory_usage': memory_usage
    }

@app.route('/api/system/stats')
def get_system_stats():
    """Get system statistics for dashboard"""
    try:
        return jsonify(_build_system_stats())
    except Exception as e:
        return jsonify({'error': str(e)}), 500

# Global activity tracking storage
activity_feed = []
activity_feed_lock = threading.Lock()

@app.route('/api/activity/feed')
def get_activity_feed():
    """Get recent activity feed for dashboard"""
    try:
        with activity_feed_lock:
            # Return last 10 activities in reverse chronological order
            return jsonify({'activities': activity_feed[-10:][::-1]})
    except Exception as e:
        return jsonify({'error': str(e)}), 500

@app.route('/api/activity/toasts')
def get_recent_toasts():
    """Get recent activities that should show toasts"""
    try:
        import time
        current_time = time.time()
        
        with activity_feed_lock:
            # Return activities from last 10 seconds that should show toasts
            recent_toasts = [
                activity for activity in activity_feed 
                if activity.get('show_toast', True) and 
                   (current_time - activity.get('timestamp', 0)) <= 10
            ]
            return jsonify({'toasts': recent_toasts})
    except Exception as e:
        return jsonify({'error': str(e)}), 500

@app.route('/api/logs')
def get_activity_logs():
    """Get formatted activity feed for display in sync page log area"""
    try:
        with activity_feed_lock:
            # Get the last 50 activities (more than the dashboard shows)
            recent_activities = activity_feed[-50:] if len(activity_feed) > 50 else activity_feed[:]

            # Reverse order so newest appears at top
            recent_activities = recent_activities[::-1]

            # Format activities as readable log entries
            formatted_logs = []

            if not recent_activities:
                formatted_logs = [
                    "No recent activity.",
                    "Sync and download operations will appear here in real-time."
                ]
            else:
                for activity in recent_activities:
                    # Format: [TIME] ICON TITLE - SUBTITLE
                    timestamp = activity.get('time', 'Unknown')
                    icon = activity.get('icon', '•')
                    title = activity.get('title', 'Activity')
                    subtitle = activity.get('subtitle', '')

                    # Create a clean, readable log entry
                    if subtitle:
                        log_entry = f"[{timestamp}] {icon} {title} - {subtitle}"
                    else:
                        log_entry = f"[{timestamp}] {icon} {title}"

                    formatted_logs.append(log_entry)

            return jsonify({'logs': formatted_logs})

    except Exception as e:
        return jsonify({'logs': [f'Error reading activity feed: {str(e)}']})

def add_activity_item(icon: str, title: str, subtitle: str, time_ago: str = "Now", show_toast: bool = True):
    """Add activity item to the feed (replicates dashboard.py functionality)"""
    try:
        import time
        activity_item = {
            'icon': icon,
            'title': title,
            'subtitle': subtitle,
            'time': time_ago,
            'timestamp': time.time(),
            'show_toast': show_toast
        }

        with activity_feed_lock:
            activity_feed.append(activity_item)
            # Keep only last 20 items to prevent memory growth
            if len(activity_feed) > 20:
                activity_feed.pop(0)

        # Instant toast push via WebSocket (replaces 3-second polling)
        if show_toast:
            try:
                socketio.emit('dashboard:toast', activity_item)
            except Exception:
                pass

        print(f"📝 Activity: {icon} {title} - {subtitle}")
    except Exception as e:
        print(f"Error adding activity item: {e}")

# --- Internal API Key Management (browser-only, no auth) ---
@app.route('/api/v1/api-keys-internal', methods=['GET'])
def list_api_keys_internal():
    """List API keys for the settings page (no auth required — same as all UI routes)."""
    keys = config_manager.get('api_keys', [])
    safe_keys = [
        {
            "id": k.get("id"),
            "label": k.get("label", ""),
            "key_prefix": k.get("key_prefix", ""),
            "created_at": k.get("created_at"),
            "last_used_at": k.get("last_used_at"),
        }
        for k in keys
    ]
    return jsonify({"success": True, "data": {"keys": safe_keys}})

@app.route('/api/v1/api-keys-internal/generate', methods=['POST'])
def generate_api_key_internal():
    """Generate API key from settings page (no auth required)."""
    from api.auth import generate_api_key
    body = request.get_json(silent=True) or {}
    label = body.get("label", "")
    raw_key, record = generate_api_key(label)
    keys = config_manager.get('api_keys', [])
    keys.append(record)
    config_manager.set('api_keys', keys)
    return jsonify({"success": True, "data": {
        "key": raw_key,
        "id": record["id"],
        "label": record["label"],
        "key_prefix": record["key_prefix"],
        "created_at": record["created_at"],
    }}), 201

@app.route('/api/v1/api-keys-internal/revoke/<key_id>', methods=['DELETE'])
def revoke_api_key_internal(key_id):
    """Revoke API key from settings page (no auth required)."""
    keys = config_manager.get('api_keys', [])
    original_len = len(keys)
    keys = [k for k in keys if k.get("id") != key_id]
    if len(keys) == original_len:
        return jsonify({"success": False, "error": {"message": "Key not found"}}), 404
    config_manager.set('api_keys', keys)
    return jsonify({"success": True, "data": {"message": "API key revoked"}})


@app.route('/api/settings', methods=['GET', 'POST'])
def handle_settings():
    global tidal_client # Declare that we might modify the global instance
    if not config_manager:
        return jsonify({"error": "Server configuration manager is not initialized."}), 500
    if request.method == 'POST':
        try:
            new_settings = request.get_json()
            if not new_settings:
                return jsonify({"success": False, "error": "No data received."}), 400

            if 'active_media_server' in new_settings:
                config_manager.set_active_media_server(new_settings['active_media_server'])

            for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'download_source', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'tidal_download', 'listenbrainz', 'acoustid', 'import', 'lossy_copy', 'ui_appearance', 'youtube']:
                if service in new_settings:
                    for key, value in new_settings[service].items():
                        config_manager.set(f'{service}.{key}', value)

            print("✅ Settings saved successfully via Web UI.")
            
            # Add activity for settings save
            changed_services = list(new_settings.keys())
            services_text = ", ".join(changed_services)
            add_activity_item("⚙️", "Settings Updated", f"{services_text} configuration saved", "Now")
            
            add_activity_item("⚙️", "Settings Updated", f"{services_text} configuration saved", "Now")
            
            spotify_client.reload_config()
            plex_client.server = None
            jellyfin_client.reload_config()
            navidrome_client.reload_config()
            # Reload orchestrator settings (download source mode, hybrid_primary, etc.)
            soulseek_client.reload_settings()
            # Reload YouTube client settings (rate limiting, cookies)
            if hasattr(soulseek_client, 'youtube'):
                soulseek_client.youtube.reload_settings()
            # FIX: Re-instantiate the global tidal_client to pick up new settings
            tidal_client = TidalClient()
            print("✅ Service clients re-initialized with new settings.")
            return jsonify({"success": True, "message": "Settings saved successfully."})
        except Exception as e:
            return jsonify({"success": False, "error": str(e)}), 500
    else:  # GET request
        try:
            return jsonify(config_manager.config_data)
        except Exception as e:
            return jsonify({"error": str(e)}), 500

dev_mode_enabled = False

@app.route('/api/dev-mode', methods=['GET', 'POST'])
def handle_dev_mode():
    global dev_mode_enabled
    if request.method == 'POST':
        data = request.get_json()
        if data.get('password') == 'hydratest':
            dev_mode_enabled = True
            print("🔧 Dev mode activated")
            return jsonify({"success": True, "enabled": True})
        return jsonify({"success": False, "error": "Invalid password"}), 401
    return jsonify({"enabled": dev_mode_enabled})

# ── Hydrabase WebSocket Connection ──
_hydrabase_ws = None
_hydrabase_lock = threading.Lock()

# ── Hydrabase Comparison Store ──
import collections as _collections
_hydrabase_comparisons = _collections.OrderedDict()
_COMPARISON_MAX_ENTRIES = 50
_comparison_lock = threading.Lock()

def _is_hydrabase_active():
    """Check if Hydrabase should be used as the primary metadata source.
    Returns False when dev mode is off — no behavior change for normal users."""
    try:
        return (dev_mode_enabled
                and hydrabase_client is not None
                and hydrabase_client.is_connected())
    except NameError:
        return False

def _run_background_comparison(query, hydrabase_counts=None):
    """Run Spotify + iTunes searches in background and store for comparison.

    Args:
        query: Search query string.
        hydrabase_counts: Optional pre-computed dict {'tracks': N, 'artists': N, 'albums': N}
                          from the primary search to avoid redundant Hydrabase round-trips.
    """
    def _worker():
        try:
            result = {'timestamp': time.time(), 'query': query}

            # Use pre-computed counts if available, otherwise fetch from Hydrabase
            if hydrabase_counts is not None:
                hydra_data = hydrabase_counts
            else:
                hydra_data = {'tracks': 0, 'artists': 0, 'albums': 0}
                if _is_hydrabase_active():
                    raw_t = hydrabase_client.search_raw(query, 'track')
                    raw_ar = hydrabase_client.search_raw(query, 'artists')
                    raw_al = hydrabase_client.search_raw(query, 'album')
                    hydra_data = {
                        'tracks': len(raw_t) if raw_t else 0,
                        'artists': len(raw_ar) if raw_ar else 0,
                        'albums': len(raw_al) if raw_al else 0
                    }
            result['hydrabase'] = hydra_data

            # Spotify results
            spotify_data = {'tracks': 0, 'artists': 0, 'albums': 0}
            if spotify_client and spotify_client.is_authenticated():
                try:
                    s_tracks = spotify_client.search_tracks(query, limit=10)
                    s_artists = spotify_client.search_artists(query, limit=10)
                    s_albums = spotify_client.search_albums(query, limit=10)
                    spotify_data = {
                        'tracks': len(s_tracks),
                        'artists': len(s_artists),
                        'albums': len(s_albums)
                    }
                except Exception as e:
                    logger.debug(f"Comparison Spotify search failed: {e}")
            result['spotify'] = spotify_data

            # iTunes results
            itunes_data = {'tracks': 0, 'artists': 0, 'albums': 0}
            try:
                from core.itunes_client import iTunesClient
                itunes = iTunesClient()
                i_tracks = itunes.search_tracks(query, limit=10)
                i_artists = itunes.search_artists(query, limit=10)
                i_albums = itunes.search_albums(query, limit=10)
                itunes_data = {
                    'tracks': len(i_tracks),
                    'artists': len(i_artists),
                    'albums': len(i_albums)
                }
            except Exception as e:
                logger.debug(f"Comparison iTunes search failed: {e}")
            result['itunes'] = itunes_data

            with _comparison_lock:
                _hydrabase_comparisons[query] = result
                while len(_hydrabase_comparisons) > _COMPARISON_MAX_ENTRIES:
                    _hydrabase_comparisons.popitem(last=False)

            logger.info(f"Background comparison stored for '{query}': H={hydra_data}, S={spotify_data}, I={itunes_data}")

        except Exception as e:
            logger.error(f"Background comparison failed for '{query}': {e}")

    threading.Thread(target=_worker, daemon=True).start()

@app.route('/api/hydrabase/connect', methods=['POST'])
def hydrabase_connect():
    """Connect to a Hydrabase instance via WebSocket."""
    global _hydrabase_ws
    if not dev_mode_enabled:
        return jsonify({"success": False, "error": "Dev mode not active"}), 403
    data = request.get_json()
    url = data.get('url', '').strip()
    api_key = data.get('api_key', '').strip()
    if not url or not api_key:
        return jsonify({"success": False, "error": "URL and API key required"}), 400
    try:
        import websocket
        with _hydrabase_lock:
            # Close existing connection if any
            if _hydrabase_ws:
                try:
                    _hydrabase_ws.close()
                except:
                    pass
            ws = websocket.create_connection(
                url,
                header={"x-api-key": api_key},
                timeout=10
            )
            _hydrabase_ws = ws
        # Save credentials for auto-reconnect
        config_manager.set('hydrabase.url', url)
        config_manager.set('hydrabase.api_key', api_key)
        config_manager.set('hydrabase.auto_connect', True)
        print(f"🧪 [Hydrabase] Connected to {url}")
        return jsonify({"success": True, "message": "Connected"})
    except Exception as e:
        print(f"⚠️ [Hydrabase] Connection failed: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/hydrabase/disconnect', methods=['POST'])
def hydrabase_disconnect():
    """Disconnect from Hydrabase and disable dev mode."""
    global _hydrabase_ws, dev_mode_enabled
    with _hydrabase_lock:
        if _hydrabase_ws:
            try:
                _hydrabase_ws.close()
            except:
                pass
            _hydrabase_ws = None
    config_manager.set('hydrabase.auto_connect', False)
    dev_mode_enabled = False
    print("🧪 [Hydrabase] Disconnected — dev mode disabled")
    return jsonify({"success": True})

@app.route('/api/hydrabase/status')
def hydrabase_status():
    """Check if connected to Hydrabase."""
    try:
        connected = _hydrabase_ws is not None and _hydrabase_ws.connected
    except Exception:
        connected = False
    hydra_config = config_manager.get_hydrabase_config()
    peer_count = None
    try:
        if hydrabase_client and hydrabase_client.last_peer_count is not None:
            peer_count = hydrabase_client.last_peer_count
    except NameError:
        pass
    return jsonify({
        "connected": connected,
        "saved_url": hydra_config.get('url', ''),
        "saved_api_key": hydra_config.get('api_key', ''),
        "auto_connect": hydra_config.get('auto_connect', False),
        "peer_count": peer_count
    })

@app.route('/api/hydrabase/comparisons')
def hydrabase_comparisons():
    """Get recent comparison results (Hydrabase vs Spotify vs iTunes)."""
    if not dev_mode_enabled:
        return jsonify({"success": False, "error": "Dev mode not active"}), 403
    with _comparison_lock:
        items = list(reversed(_hydrabase_comparisons.values()))
    return jsonify({"success": True, "comparisons": items})

@app.route('/api/hydrabase/send', methods=['POST'])
def hydrabase_send():
    """Send a raw JSON payload to Hydrabase and return the response."""
    global _hydrabase_ws
    if not dev_mode_enabled:
        return jsonify({"success": False, "error": "Dev mode not active"}), 403
    if not _hydrabase_ws or not _hydrabase_ws.connected:
        return jsonify({"success": False, "error": "Not connected to Hydrabase"}), 400
    data = request.get_json()
    payload = data.get('payload')
    if not payload:
        return jsonify({"success": False, "error": "No payload provided"}), 400
    try:
        message = json.dumps(payload) if isinstance(payload, dict) else str(payload)
        with _hydrabase_lock:
            _hydrabase_ws.send(message)
            response = _hydrabase_ws.recv()
        try:
            result = json.loads(response)
        except json.JSONDecodeError:
            result = response
        print(f"🧪 [Hydrabase] Sent payload — got response")
        return jsonify({"success": True, "data": result})
    except Exception as e:
        print(f"⚠️ [Hydrabase] Send failed: {e}")
        with _hydrabase_lock:
            try:
                _hydrabase_ws.close()
            except:
                pass
            _hydrabase_ws = None
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/settings/log-level', methods=['GET', 'POST'])
def handle_log_level():
    """Get or set the application log level"""
    from utils.logging_config import set_log_level, get_current_log_level
    from database.music_database import MusicDatabase

    if request.method == 'POST':
        try:
            data = request.get_json()
            level = data.get('level')

            if not level or level.upper() not in ['DEBUG', 'INFO', 'WARNING', 'ERROR']:
                return jsonify({"success": False, "error": "Invalid log level. Must be DEBUG, INFO, WARNING, or ERROR"}), 400

            # Change the log level dynamically
            success = set_log_level(level)

            if success:
                # Save to database preferences
                db = MusicDatabase()
                db.set_preference('log_level', level.upper())

                logger.info(f"Log level changed to {level.upper()} via Web UI")
                add_activity_item("🔍", "Log Level Changed", f"Set to {level.upper()}", "Now")

                return jsonify({"success": True, "level": level.upper()})
            else:
                return jsonify({"success": False, "error": "Failed to set log level"}), 500

        except Exception as e:
            logger.error(f"Error setting log level: {e}")
            return jsonify({"success": False, "error": str(e)}), 500

    else:  # GET request
        try:
            current_level = get_current_log_level()
            return jsonify({"success": True, "level": current_level})
        except Exception as e:
            logger.error(f"Error getting log level: {e}")
            return jsonify({"success": False, "error": str(e)}), 500

# ===========================
# AUTOMATIONS API
# ===========================

@app.route('/api/automations', methods=['GET'])
def list_automations():
    """List all automations for the current profile."""
    try:
        profile_id = session.get('profile_id', 1)
        db = get_database()
        automations = db.get_automations(profile_id)
        # Parse JSON config fields for frontend
        for auto in automations:
            for field in ('trigger_config', 'action_config', 'notify_config', 'last_result'):
                try:
                    auto[field] = json.loads(auto[field]) if isinstance(auto[field], str) else auto[field]
                except (json.JSONDecodeError, TypeError):
                    if field in ('trigger_config', 'action_config', 'notify_config'):
                        auto[field] = {}
                    else:
                        auto[field] = None
        return jsonify(automations)
    except Exception as e:
        logger.error(f"Error listing automations: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/automations', methods=['POST'])
def create_automation():
    """Create a new automation."""
    try:
        data = request.get_json()
        name = data.get('name', '').strip()
        if not name:
            return jsonify({"error": "Name is required"}), 400

        trigger_type = data.get('trigger_type', 'schedule')
        trigger_config = json.dumps(data.get('trigger_config', {}))
        action_type = data.get('action_type', 'process_wishlist')
        action_config = json.dumps(data.get('action_config', {}))
        notify_type = data.get('notify_type') or None
        notify_config = json.dumps(data.get('notify_config', {})) if notify_type else '{}'
        profile_id = session.get('profile_id', 1)

        db = get_database()
        auto_id = db.create_automation(name, trigger_type, trigger_config, action_type, action_config, profile_id, notify_type, notify_config)
        if auto_id is None:
            return jsonify({"error": "Failed to create automation"}), 500

        # Schedule it
        if automation_engine:
            automation_engine.schedule_automation(auto_id)

        return jsonify({"success": True, "id": auto_id})
    except Exception as e:
        logger.error(f"Error creating automation: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/automations/<int:automation_id>', methods=['GET'])
def get_automation(automation_id):
    """Get a single automation."""
    try:
        db = get_database()
        auto = db.get_automation(automation_id)
        if not auto:
            return jsonify({"error": "Automation not found"}), 404
        for field in ('trigger_config', 'action_config', 'notify_config', 'last_result'):
            try:
                auto[field] = json.loads(auto[field]) if isinstance(auto[field], str) else auto[field]
            except (json.JSONDecodeError, TypeError):
                if field in ('trigger_config', 'action_config', 'notify_config'):
                    auto[field] = {}
                else:
                    auto[field] = None
        return jsonify(auto)
    except Exception as e:
        logger.error(f"Error getting automation: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/automations/<int:automation_id>', methods=['PUT'])
def update_automation_endpoint(automation_id):
    """Update an automation."""
    try:
        data = request.get_json()
        db = get_database()

        update_fields = {}
        if 'name' in data:
            update_fields['name'] = data['name'].strip()
        if 'trigger_type' in data:
            update_fields['trigger_type'] = data['trigger_type']
        if 'trigger_config' in data:
            update_fields['trigger_config'] = json.dumps(data['trigger_config'])
        if 'action_type' in data:
            update_fields['action_type'] = data['action_type']
        if 'action_config' in data:
            update_fields['action_config'] = json.dumps(data['action_config'])
        if 'notify_type' in data:
            update_fields['notify_type'] = data['notify_type'] or None
        if 'notify_config' in data:
            update_fields['notify_config'] = json.dumps(data['notify_config'])

        if not update_fields:
            return jsonify({"error": "No fields to update"}), 400

        success = db.update_automation(automation_id, **update_fields)
        if not success:
            return jsonify({"error": "Automation not found"}), 404

        # Reschedule
        if automation_engine:
            auto = db.get_automation(automation_id)
            if auto and auto.get('enabled'):
                automation_engine.schedule_automation(automation_id)
            else:
                automation_engine.cancel_automation(automation_id)

        return jsonify({"success": True})
    except Exception as e:
        logger.error(f"Error updating automation: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/automations/<int:automation_id>', methods=['DELETE'])
def delete_automation_endpoint(automation_id):
    """Delete an automation. System automations cannot be deleted."""
    try:
        db = get_database()
        auto = db.get_automation(automation_id)
        if auto and auto.get('is_system'):
            return jsonify({"error": "System automations cannot be deleted"}), 403
        if automation_engine:
            automation_engine.cancel_automation(automation_id)
        success = db.delete_automation(automation_id)
        if not success:
            return jsonify({"error": "Automation not found"}), 404
        return jsonify({"success": True})
    except Exception as e:
        logger.error(f"Error deleting automation: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/automations/<int:automation_id>/toggle', methods=['POST'])
def toggle_automation_endpoint(automation_id):
    """Toggle an automation's enabled state."""
    try:
        db = get_database()
        success = db.toggle_automation(automation_id)
        if not success:
            return jsonify({"error": "Automation not found"}), 404

        # Reschedule or cancel based on new state
        if automation_engine:
            auto = db.get_automation(automation_id)
            if auto and auto.get('enabled'):
                automation_engine.schedule_automation(automation_id)
            else:
                automation_engine.cancel_automation(automation_id)

        return jsonify({"success": True})
    except Exception as e:
        logger.error(f"Error toggling automation: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/automations/<int:automation_id>/run', methods=['POST'])
def run_automation_endpoint(automation_id):
    """Manually trigger an automation."""
    try:
        if not automation_engine:
            return jsonify({"error": "Automation engine not available"}), 500
        success = automation_engine.run_now(automation_id)
        if not success:
            return jsonify({"error": "Automation not found"}), 404
        return jsonify({"success": True})
    except Exception as e:
        logger.error(f"Error running automation: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/automations/progress', methods=['GET'])
def get_automation_progress():
    """Get current progress state for all running/recently finished automations."""
    try:
        with automation_progress_lock:
            result = {}
            for aid, state in automation_progress_states.items():
                if state['status'] in ('running', 'finished', 'error'):
                    cp = dict(state)
                    cp['log'] = list(state['log'])
                    result[str(aid)] = cp
        return jsonify(result)
    except Exception as e:
        return jsonify({"error": str(e)}), 500

@app.route('/api/automations/blocks', methods=['GET'])
def get_automation_blocks():
    """Return available block types for the automation builder sidebar."""
    return jsonify({
        "triggers": [
            {"type": "schedule", "label": "Schedule", "icon": "clock", "description": "Run on a timer interval", "available": True,
             "config_fields": [
                 {"key": "interval", "type": "number", "label": "Every", "default": 6, "min": 1},
                 {"key": "unit", "type": "select", "label": "Unit",
                  "options": [{"value": "minutes", "label": "Minutes"}, {"value": "hours", "label": "Hours"}, {"value": "days", "label": "Days"}],
                  "default": "hours"}
             ]},
            {"type": "daily_time", "label": "Daily Time", "icon": "clock", "description": "Run every day at a specific time", "available": True,
             "config_fields": [
                 {"key": "time", "type": "time", "label": "At", "default": "03:00"}
             ]},
            {"type": "weekly_time", "label": "Weekly Schedule", "icon": "calendar", "description": "Run on specific days of the week at a set time", "available": True,
             "config_fields": [
                 {"key": "time", "type": "time", "label": "At", "default": "03:00"},
                 {"key": "days", "type": "multi_select", "label": "Days",
                  "options": [{"value": "mon", "label": "Mon"}, {"value": "tue", "label": "Tue"}, {"value": "wed", "label": "Wed"},
                              {"value": "thu", "label": "Thu"}, {"value": "fri", "label": "Fri"}, {"value": "sat", "label": "Sat"}, {"value": "sun", "label": "Sun"}]}
             ]},
            {"type": "app_started", "label": "App Started", "icon": "power", "description": "When SoulSync starts up", "available": True},
            {"type": "track_downloaded", "label": "Track Downloaded", "icon": "download", "description": "When a track finishes downloading", "available": True,
             "has_conditions": True,
             "condition_fields": ["artist", "title", "album", "quality"],
             "variables": ["artist", "title", "album", "quality"]},
            {"type": "batch_complete", "label": "Batch Complete", "icon": "check-circle", "description": "When an album/playlist download finishes", "available": True,
             "has_conditions": True,
             "condition_fields": ["playlist_name"],
             "variables": ["playlist_name", "total_tracks", "completed_tracks", "failed_tracks"]},
            {"type": "watchlist_new_release", "label": "New Release Found", "icon": "bell", "description": "When watchlist detects new music", "available": True,
             "has_conditions": True,
             "condition_fields": ["artist"],
             "variables": ["artist", "new_tracks", "added_to_wishlist"]},
            {"type": "playlist_synced", "label": "Playlist Synced", "icon": "refresh", "description": "When a playlist sync completes", "available": True,
             "has_conditions": True,
             "condition_fields": ["playlist_name"],
             "variables": ["playlist_name", "total_tracks", "matched_tracks", "synced_tracks", "failed_tracks"]},
            {"type": "playlist_changed", "label": "Playlist Changed", "icon": "edit", "description": "When a mirrored playlist detects track changes from source", "available": True,
             "has_conditions": True,
             "condition_fields": ["playlist_name"],
             "variables": ["playlist_name", "old_count", "new_count", "added", "removed"]},
            {"type": "discovery_completed", "label": "Discovery Complete", "icon": "search", "description": "When playlist track discovery finishes", "available": True,
             "has_conditions": True,
             "condition_fields": ["playlist_name"],
             "variables": ["playlist_name", "total_tracks", "discovered_count", "failed_count", "skipped_count"]},
            # Phase 3 triggers
            {"type": "wishlist_processing_completed", "label": "Wishlist Processed", "icon": "check-circle",
             "description": "When auto-wishlist processing finishes", "available": True,
             "variables": ["tracks_processed", "tracks_found", "tracks_failed"]},
            {"type": "watchlist_scan_completed", "label": "Watchlist Scan Done", "icon": "check-circle",
             "description": "When watchlist scan finishes", "available": True,
             "variables": ["artists_scanned", "new_tracks_found", "tracks_added"]},
            {"type": "database_update_completed", "label": "Database Updated", "icon": "database",
             "description": "When library database refresh finishes", "available": True,
             "variables": ["total_artists", "total_albums", "total_tracks"]},
            {"type": "download_failed", "label": "Download Failed", "icon": "x-circle",
             "description": "When a track permanently fails to download", "available": True,
             "has_conditions": True, "condition_fields": ["artist", "title", "reason"],
             "variables": ["artist", "title", "reason"]},
            {"type": "download_quarantined", "label": "File Quarantined", "icon": "alert-triangle",
             "description": "When AcoustID verification fails", "available": True,
             "has_conditions": True, "condition_fields": ["artist", "title"],
             "variables": ["artist", "title", "reason"]},
            {"type": "wishlist_item_added", "label": "Wishlist Item Added", "icon": "plus-circle",
             "description": "When a track is added to wishlist", "available": True,
             "has_conditions": True, "condition_fields": ["artist", "title"],
             "variables": ["artist", "title", "reason"]},
            {"type": "watchlist_artist_added", "label": "Artist Watched", "icon": "user-plus",
             "description": "When an artist is added to watchlist", "available": True,
             "has_conditions": True, "condition_fields": ["artist"],
             "variables": ["artist", "artist_id"]},
            {"type": "watchlist_artist_removed", "label": "Artist Unwatched", "icon": "user-minus",
             "description": "When an artist is removed from watchlist", "available": True,
             "has_conditions": True, "condition_fields": ["artist"],
             "variables": ["artist", "artist_id"]},
            {"type": "import_completed", "label": "Import Complete", "icon": "upload",
             "description": "When album/track import finishes", "available": True,
             "has_conditions": True, "condition_fields": ["artist", "album_name"],
             "variables": ["track_count", "album_name", "artist"]},
            {"type": "mirrored_playlist_created", "label": "Playlist Mirrored", "icon": "copy",
             "description": "When a new playlist is mirrored", "available": True,
             "has_conditions": True, "condition_fields": ["playlist_name", "source"],
             "variables": ["playlist_name", "source", "track_count"]},
            {"type": "quality_scan_completed", "label": "Quality Scan Done", "icon": "bar-chart",
             "description": "When quality scan finishes", "available": True,
             "variables": ["quality_met", "low_quality", "total_scanned"]},
            {"type": "duplicate_scan_completed", "label": "Duplicate Scan Done", "icon": "layers",
             "description": "When duplicate cleaner finishes", "available": True,
             "variables": ["files_scanned", "duplicates_found", "space_freed"]},
        ],
        "actions": [
            {"type": "process_wishlist", "label": "Process Wishlist", "icon": "list", "description": "Retry failed downloads from wishlist", "available": True,
             "config_fields": [{"key": "category", "type": "select", "label": "Category", "options": [{"value": "all", "label": "All"}, {"value": "albums", "label": "Albums"}, {"value": "singles", "label": "Singles"}], "default": "all"}]},
            {"type": "scan_watchlist", "label": "Scan Watchlist", "icon": "eye", "description": "Check watched artists for new releases", "available": True},
            {"type": "scan_library", "label": "Scan Library", "icon": "refresh", "description": "Trigger media server library scan", "available": True},
            {"type": "refresh_mirrored", "label": "Refresh Mirrored Playlist", "icon": "copy", "description": "Re-fetch playlist from source and update mirror", "available": True,
             "config_fields": [
                 {"key": "playlist_id", "type": "mirrored_playlist_select", "label": "Playlist"},
                 {"key": "all", "type": "checkbox", "label": "Refresh all mirrored playlists", "default": False}
             ]},
            {"type": "sync_playlist", "label": "Sync Playlist", "icon": "sync", "description": "Sync mirrored playlist to media server", "available": True,
             "config_fields": [
                 {"key": "playlist_id", "type": "mirrored_playlist_select", "label": "Playlist"}
             ]},
            {"type": "discover_playlist", "label": "Discover Playlist", "icon": "search", "description": "Find official Spotify/iTunes metadata for mirrored playlist tracks", "available": True,
             "config_fields": [
                 {"key": "playlist_id", "type": "mirrored_playlist_select", "label": "Playlist"},
                 {"key": "all", "type": "checkbox", "label": "Discover all mirrored playlists", "default": False}
             ]},
            {"type": "notify_only", "label": "Notify Only", "icon": "bell", "description": "No action — just send notification", "available": True},
            # Phase 3 actions
            {"type": "start_database_update", "label": "Update Database", "icon": "database",
             "description": "Trigger library database refresh", "available": True,
             "config_fields": [
                 {"key": "full_refresh", "type": "checkbox", "label": "Full refresh (slower)", "default": False}
             ]},
            {"type": "run_duplicate_cleaner", "label": "Run Duplicate Cleaner", "icon": "layers",
             "description": "Scan for and remove duplicate files", "available": True},
            {"type": "clear_quarantine", "label": "Clear Quarantine", "icon": "trash",
             "description": "Delete all quarantined files", "available": True},
            {"type": "cleanup_wishlist", "label": "Clean Up Wishlist", "icon": "filter",
             "description": "Remove duplicate/owned tracks from wishlist", "available": True},
            {"type": "update_discovery_pool", "label": "Update Discovery", "icon": "compass",
             "description": "Refresh discovery pool with new tracks", "available": True},
            {"type": "start_quality_scan", "label": "Run Quality Scan", "icon": "bar-chart",
             "description": "Scan for low-quality audio files", "available": True,
             "config_fields": [
                 {"key": "scope", "type": "select", "label": "Scope",
                  "options": [{"value": "watchlist", "label": "Watchlist Artists"}, {"value": "library", "label": "Full Library"}],
                  "default": "watchlist"}
             ]},
            {"type": "backup_database", "label": "Backup Database", "icon": "save",
             "description": "Create timestamped database backup", "available": True},
        ],
        "notifications": [
            {"type": "discord_webhook", "label": "Discord Webhook", "icon": "message", "description": "Send a Discord notification", "available": True,
             "variables": ["time", "name", "run_count", "status"]},
            {"type": "pushbullet", "label": "Pushbullet", "icon": "push", "description": "Push notification to phone/desktop", "available": True,
             "variables": ["time", "name", "run_count", "status"]},
            {"type": "telegram", "label": "Telegram", "icon": "message", "description": "Send a Telegram message", "available": True,
             "variables": ["time", "name", "run_count", "status"]},
        ]
    })

@app.route('/api/mirrored-playlists/list', methods=['GET'])
def get_mirrored_playlists_list():
    """Return simple list of mirrored playlists for automation config dropdowns."""
    try:
        database = get_database()
        profile_id = get_current_profile_id()
        playlists = database.get_mirrored_playlists(profile_id=profile_id)
        return jsonify([{"id": p['id'], "name": p['name']} for p in playlists])
    except Exception as e:
        return jsonify([]), 200

@app.route('/api/test-connection', methods=['POST'])
def test_connection_endpoint():
    data = request.get_json()
    service = data.get('service')
    if not service:
        return jsonify({"success": False, "error": "No service specified."}), 400

    print(f"Received test connection request for: {service}")
    
    # Get the current settings from the main config manager to test with
    test_config = config_manager.get(service, {})
    
    # For media servers, the service name might be 'server'
    if service == 'server':
        active_server = config_manager.get_active_media_server()
        test_config = config_manager.get(active_server, {})
        service = active_server # use the actual server name for the test

    success, message = run_service_test(service, test_config)

    # Update status cache immediately when test succeeds to reflect current state
    import time
    if success:
        current_time = time.time()
        if service == 'spotify':
            _status_cache['spotify']['connected'] = True
            _status_cache['spotify']['source'] = 'spotify' if spotify_client.is_spotify_authenticated() else 'itunes'
            _status_cache_timestamps['spotify'] = current_time
            print("✅ Updated Spotify status cache after successful test")
        elif service in ['plex', 'jellyfin', 'navidrome']:
            _status_cache['media_server']['connected'] = True
            _status_cache['media_server']['type'] = service
            _status_cache_timestamps['media_server'] = current_time
            print(f"✅ Updated {service} status cache after successful test")
        elif service == 'soulseek':
            _status_cache['soulseek']['connected'] = True
            _status_cache_timestamps['soulseek'] = current_time
            print("✅ Updated Soulseek status cache after successful test")
        elif service == 'listenbrainz':
            print("✅ ListenBrainz test successful")

    # Add activity for connection test
    if success:
        add_activity_item("✅", "Connection Test", message, "Now")
    else:
        add_activity_item("❌", "Connection Test", f"{service.title()} connection failed", "Now")

    return jsonify({"success": success, "error": "" if success else message, "message": message if success else ""})

@app.route('/api/test-dashboard-connection', methods=['POST'])
def test_dashboard_connection_endpoint():
    """Test connection from dashboard - creates specific dashboard activity items"""
    data = request.get_json()
    service = data.get('service')
    if not service:
        return jsonify({"success": False, "error": "No service specified."}), 400

    print(f"Received dashboard test connection request for: {service}")
    
    # Get the current settings from the main config manager to test with
    test_config = config_manager.get(service, {})
    
    # For media servers, the service name might be 'server'
    if service == 'server':
        active_server = config_manager.get_active_media_server()
        test_config = config_manager.get(active_server, {})
        service = active_server # use the actual server name for the test

    success, message = run_service_test(service, test_config)

    # Update status cache immediately when test succeeds to reflect current state
    import time
    if success:
        current_time = time.time()
        if service == 'spotify':
            _status_cache['spotify']['connected'] = True
            _status_cache['spotify']['source'] = 'spotify' if spotify_client.is_spotify_authenticated() else 'itunes'
            _status_cache_timestamps['spotify'] = current_time
            print("✅ Updated Spotify status cache after successful dashboard test")
        elif service in ['plex', 'jellyfin', 'navidrome']:
            _status_cache['media_server']['connected'] = True
            _status_cache['media_server']['type'] = service
            _status_cache_timestamps['media_server'] = current_time
            print(f"✅ Updated {service} status cache after successful dashboard test")
        elif service == 'soulseek':
            _status_cache['soulseek']['connected'] = True
            _status_cache_timestamps['soulseek'] = current_time
            print("✅ Updated Soulseek status cache after successful dashboard test")

    # Add activity for dashboard connection test (different from settings test)
    if success:
        add_activity_item("🎛️", "Dashboard Test", message, "Now")
    else:
        add_activity_item("⚠️", "Dashboard Test", f"{service.title()} service check failed", "Now")

    return jsonify({"success": success, "error": "" if success else message, "message": message if success else ""})

@app.route('/api/detect-media-server', methods=['POST'])
def detect_media_server_endpoint():
    data = request.get_json()
    server_type = data.get('server_type')
    print(f"Received auto-detect request for: {server_type}")
    
    # Add activity for auto-detect start
    add_activity_item("🔍", "Auto-Detect Started", f"Searching for {server_type} server", "Now")
    
    found_url = run_detection(server_type)
    if found_url:
        add_activity_item("✅", "Auto-Detect Complete", f"{server_type} found at {found_url}", "Now")
        return jsonify({"success": True, "found_url": found_url})
    else:
        add_activity_item("❌", "Auto-Detect Failed", f"No {server_type} server found", "Now")
        return jsonify({"success": False, "error": f"No {server_type} server found on common local addresses."})

@app.route('/api/plex/music-libraries', methods=['GET'])
def get_plex_music_libraries():
    """Get list of all available music libraries from Plex"""
    try:
        libraries = plex_client.get_available_music_libraries()

        # Get currently selected library
        from database.music_database import MusicDatabase
        db = MusicDatabase()
        selected_library = db.get_preference('plex_music_library')

        # Get the currently active library name
        current_library = None
        if plex_client.music_library:
            current_library = plex_client.music_library.title

        return jsonify({
            "success": True,
            "libraries": libraries,
            "selected": selected_library,
            "current": current_library
        })
    except Exception as e:
        logger.error(f"Error getting Plex music libraries: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/plex/select-music-library', methods=['POST'])
def select_plex_music_library():
    """Set the active Plex music library"""
    try:
        data = request.get_json()
        library_name = data.get('library_name')

        if not library_name:
            return jsonify({"success": False, "error": "No library name provided"}), 400

        success = plex_client.set_music_library_by_name(library_name)

        if success:
            add_activity_item("📚", "Library Selected", f"Plex music library set to: {library_name}", "Now")
            return jsonify({"success": True, "message": f"Music library set to: {library_name}"})
        else:
            return jsonify({"success": False, "error": f"Library '{library_name}' not found"}), 404

    except Exception as e:
        logger.error(f"Error setting Plex music library: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/jellyfin/users', methods=['GET'])
def get_jellyfin_users():
    """Get list of Jellyfin users that have music libraries"""
    try:
        users = jellyfin_client.get_available_users()

        # Get currently selected user
        from database.music_database import MusicDatabase
        db = MusicDatabase()
        selected_user = db.get_preference('jellyfin_user')

        # Determine the current user name from user_id
        current_user = None
        if jellyfin_client.user_id:
            for u in users:
                if u['id'] == jellyfin_client.user_id:
                    current_user = u['name']
                    break

        return jsonify({
            "success": True,
            "users": users,
            "selected": selected_user,
            "current": current_user
        })
    except Exception as e:
        logger.error(f"Error getting Jellyfin users: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/jellyfin/select-user', methods=['POST'])
def select_jellyfin_user():
    """Set the active Jellyfin user"""
    try:
        data = request.get_json()
        username = data.get('username')

        if not username:
            return jsonify({"success": False, "error": "No username provided"}), 400

        success = jellyfin_client.set_user_by_name(username)

        if success:
            add_activity_item("👤", "User Selected", f"Jellyfin user set to: {username}", "Now")
            return jsonify({"success": True, "message": f"User set to: {username}"})
        else:
            return jsonify({"success": False, "error": f"User '{username}' not found or has no music library"}), 404

    except Exception as e:
        logger.error(f"Error setting Jellyfin user: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/jellyfin/music-libraries', methods=['GET'])
def get_jellyfin_music_libraries():
    """Get list of all available music libraries from Jellyfin"""
    try:
        libraries = jellyfin_client.get_available_music_libraries()

        # Get currently selected library
        from database.music_database import MusicDatabase
        db = MusicDatabase()
        selected_library = db.get_preference('jellyfin_music_library')

        # Get the currently active library name (match Plex behavior)
        current_library = None
        if jellyfin_client.music_library_id:
            # Look up library name from ID
            for lib in libraries:
                if lib['key'] == jellyfin_client.music_library_id:
                    current_library = lib['title']
                    break

        return jsonify({
            "success": True,
            "libraries": libraries,
            "selected": selected_library,
            "current": current_library
        })
    except Exception as e:
        logger.error(f"Error getting Jellyfin music libraries: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/jellyfin/select-music-library', methods=['POST'])
def select_jellyfin_music_library():
    """Set the active Jellyfin music library"""
    try:
        data = request.get_json()
        library_name = data.get('library_name')

        if not library_name:
            return jsonify({"success": False, "error": "No library name provided"}), 400

        success = jellyfin_client.set_music_library_by_name(library_name)

        if success:
            add_activity_item("📚", "Library Selected", f"Jellyfin music library set to: {library_name}", "Now")
            return jsonify({"success": True, "message": f"Music library set to: {library_name}"})
        else:
            return jsonify({"success": False, "error": f"Library '{library_name}' not found"}), 404

    except Exception as e:
        logger.error(f"Error setting Jellyfin music library: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/navidrome/music-folders', methods=['GET'])
def get_navidrome_music_folders():
    """Get list of available music folders from Navidrome"""
    try:
        if not navidrome_client:
            return jsonify({"success": False, "error": "Navidrome client not configured"}), 400

        folders = navidrome_client.get_music_folders()

        from database.music_database import MusicDatabase
        db = MusicDatabase()
        selected_folder = db.get_preference('navidrome_music_folder')

        current_folder = None
        if navidrome_client.music_folder_id:
            for f in folders:
                if f['key'] == navidrome_client.music_folder_id:
                    current_folder = f['title']
                    break

        return jsonify({
            "success": True,
            "folders": folders,
            "selected": selected_folder,
            "current": current_folder
        })
    except Exception as e:
        logger.error(f"Error getting Navidrome music folders: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/navidrome/select-music-folder', methods=['POST'])
def select_navidrome_music_folder():
    """Set the active Navidrome music folder"""
    try:
        data = request.get_json()
        folder_name = data.get('folder_name', '')

        success = navidrome_client.set_music_folder_by_name(folder_name)

        if success:
            if folder_name:
                add_activity_item("📚", "Library Selected", f"Navidrome music folder set to: {folder_name}", "Now")
                return jsonify({"success": True, "message": f"Music folder set to: {folder_name}"})
            else:
                add_activity_item("📚", "Library Selection Cleared", "Navidrome will use all libraries", "Now")
                return jsonify({"success": True, "message": "Music folder selection cleared — using all libraries"})
        else:
            return jsonify({"success": False, "error": f"Folder '{folder_name}' not found"}), 404

    except Exception as e:
        logger.error(f"Error setting Navidrome music folder: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

# ===============================
# == QUALITY PROFILE API       ==
# ===============================

@app.route('/api/quality-profile', methods=['GET'])
def get_quality_profile():
    """Get current quality profile configuration"""
    try:
        from database.music_database import MusicDatabase
        db = MusicDatabase()
        profile = db.get_quality_profile()

        return jsonify({
            "success": True,
            "profile": profile
        })
    except Exception as e:
        logger.error(f"Error getting quality profile: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/quality-profile', methods=['POST'])
def save_quality_profile():
    """Save quality profile configuration"""
    try:
        from database.music_database import MusicDatabase
        db = MusicDatabase()

        data = request.get_json()
        if not data:
            return jsonify({"success": False, "error": "No profile data provided"}), 400

        success = db.set_quality_profile(data)

        if success:
            add_activity_item("🎵", "Quality Profile Updated", f"Preset: {data.get('preset', 'custom')}", "Now")
            return jsonify({"success": True, "message": "Quality profile saved successfully"})
        else:
            return jsonify({"success": False, "error": "Failed to save quality profile"}), 500

    except Exception as e:
        logger.error(f"Error saving quality profile: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/quality-profile/presets', methods=['GET'])
def get_quality_presets():
    """Get all available quality presets"""
    try:
        from database.music_database import MusicDatabase
        db = MusicDatabase()

        presets = {
            "audiophile": db.get_quality_preset("audiophile"),
            "balanced": db.get_quality_preset("balanced"),
            "space_saver": db.get_quality_preset("space_saver")
        }

        return jsonify({
            "success": True,
            "presets": presets
        })
    except Exception as e:
        logger.error(f"Error getting quality presets: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/quality-profile/preset/<preset_name>', methods=['POST'])
def apply_quality_preset(preset_name):
    """Apply a predefined quality preset"""
    try:
        from database.music_database import MusicDatabase
        db = MusicDatabase()

        preset = db.get_quality_preset(preset_name)
        success = db.set_quality_profile(preset)

        if success:
            add_activity_item("🎵", "Quality Preset Applied", f"Applied '{preset_name}' preset", "Now")
            return jsonify({
                "success": True,
                "message": f"Applied '{preset_name}' preset",
                "profile": preset
            })
        else:
            return jsonify({"success": False, "error": "Failed to apply preset"}), 500

    except Exception as e:
        logger.error(f"Error applying quality preset: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

# ===============================
# == END QUALITY PROFILE API   ==
# ===============================

@app.route('/api/detect-soulseek', methods=['POST'])
def detect_soulseek_endpoint():
    print("Received auto-detect request for slskd")

    # Add activity for soulseek auto-detect start
    add_activity_item("🔍", "Auto-Detect Started", "Searching for slskd server", "Now")
    found_url = run_detection('slskd')
    if found_url:
        add_activity_item("✅", "Auto-Detect Complete", f"slskd found at {found_url}", "Now")
        return jsonify({"success": True, "found_url": found_url})
    else:
        add_activity_item("❌", "Auto-Detect Failed", "No slskd server found", "Now")
        return jsonify({"success": False, "error": "No slskd server found on common local addresses."})

# --- Authentication Routes ---

@app.route('/auth/spotify')
def auth_spotify():
    """
    Initiates Spotify OAuth authentication flow
    """
    try:
        # Create a fresh spotify client to trigger OAuth
        temp_spotify_client = SpotifyClient()
        if temp_spotify_client.sp and temp_spotify_client.sp.auth_manager:
            # Get the authorization URL
            auth_url = temp_spotify_client.sp.auth_manager.get_authorize_url()
            configured_uri = config_manager.get_spotify_config().get('redirect_uri', 'http://127.0.0.1:8888/callback')
            print(f"🎵 Spotify auth initiated — redirect_uri: {configured_uri}")
            add_activity_item("🔐", "Spotify Auth Started", "Please complete OAuth in browser", "Now")

            # Detect if accessing remotely
            host = request.host.split(':')[0]
            is_remote = host not in ['127.0.0.1', 'localhost']
            is_docker = os.path.exists('/.dockerenv')

            # If in Docker and accessing via 127.0.0.1, recommend localhost
            if is_docker and host == '127.0.0.1':
                host = 'localhost'

            # Check if the redirect_uri uses port 8008 (main app) vs 8888 (standalone)
            uses_main_port = ':8008' in configured_uri or ':8888' not in configured_uri

            if is_remote or is_docker:
                # Show instructions for remote/docker access
                if uses_main_port:
                    # redirect_uri already points to port 8008 or a custom domain —
                    # callback will come through the main Flask app, no manual steps needed
                    return f'''
                    <html>
                    <head>
                        <style>
                            body {{ font-family: Arial, sans-serif; padding: 20px; max-width: 800px; margin: 0 auto; }}
                            code {{ background: #f0f0f0; padding: 4px 8px; border-radius: 3px; }}
                            .info {{ background: #e3f2fd; border-left: 4px solid #2196F3; padding: 12px 16px; margin: 16px 0; border-radius: 4px; }}
                        </style>
                    </head>
                    <body>
                        <h1>🔐 Spotify Authentication</h1>
                        <p>Click the link below to authenticate with Spotify:</p>
                        <p><a href="{auth_url}" target="_blank" style="font-size: 18px; color: #1DB954;">Authenticate with Spotify</a></p>
                        <div class="info">
                            <strong>Redirect URI:</strong> <code>{configured_uri}</code><br>
                            <small>After authorizing, Spotify will redirect back automatically. Make sure this URL matches your Spotify Dashboard redirect URI.</small>
                        </div>
                        <p>After authentication completes, you can close this window and return to SoulSync.</p>
                    </body>
                    </html>
                    '''
                else:
                    # redirect_uri still points to port 8888 — show manual steps AND suggest switching
                    return f'''
                    <html>
                    <head>
                        <style>
                            body {{ font-family: Arial, sans-serif; padding: 20px; max-width: 800px; margin: 0 auto; }}
                            code {{ background: #f0f0f0; padding: 10px; display: block; margin: 10px 0; }}
                            .highlight {{ background: #e8f5e9; }}
                            .warning {{ background: #fff3e0; border-left: 4px solid #ff9800; padding: 12px 16px; margin: 16px 0; border-radius: 4px; }}
                            .copy-btn {{
                                background: #1DB954;
                                color: white;
                                border: none;
                                padding: 8px 16px;
                                cursor: pointer;
                                border-radius: 4px;
                                font-size: 14px;
                                margin-left: 10px;
                            }}
                            .copy-btn:hover {{ background: #1ed760; }}
                            .copied {{ background: #4CAF50 !important; }}
                        </style>
                    </head>
                    <body>
                        <h1>🔐 Spotify Authentication (Remote/Docker)</h1>

                        <div class="warning">
                            <strong>Using a reverse proxy?</strong> Your redirect URI is set to <code style="display:inline; padding: 2px 6px;">{configured_uri}</code>
                            which uses port 8888. If you're behind a reverse proxy (Caddy, Nginx, Traefik), change the
                            redirect URI in SoulSync settings to use your proxy URL on the main port instead, e.g.:<br>
                            <code style="display:inline; padding: 2px 6px; background: #e8f5e9;">https://{host}/callback</code><br>
                            Then update the same URI in your <a href="https://developer.spotify.com/dashboard" target="_blank">Spotify Dashboard</a>.
                            This avoids the need for manual URL editing below.
                        </div>

                        <p><strong>Step 1:</strong> Click the link below to authenticate with Spotify</p>
                        <p><a href="{auth_url}" target="_blank" style="font-size: 18px; color: #1DB954;">{auth_url}</a></p>
                        <hr>
                        <p><strong>Step 2:</strong> After authorizing, you'll see a blank page. The URL will look like:</p>
                        <code>http://127.0.0.1:8888/callback?code=...</code>
                        <p><strong>Step 3:</strong> Change <code style="display: inline; background: #ffe6e6; padding: 2px 6px;">127.0.0.1</code> to <code style="display: inline; background: #e8f5e9; padding: 2px 6px;">{host}</code> and press Enter:
                            <button class="copy-btn" onclick="copyIP()">Copy IP</button>
                        </p>
                        <code class="highlight">http://{host}:8888/callback?code=...</code>
                        <p>Authentication will then complete!</p>

                        <script>
                            function copyIP() {{
                                navigator.clipboard.writeText('{host}').then(() => {{
                                    const btn = event.target;
                                    btn.textContent = 'Copied!';
                                    btn.classList.add('copied');
                                    setTimeout(() => {{
                                        btn.textContent = 'Copy IP';
                                        btn.classList.remove('copied');
                                    }}, 2000);
                                }});
                            }}
                        </script>
                    </body>
                    </html>
                    '''
            else:
                # Local access - simple message
                return f'<h1>🔐 Spotify Authentication</h1><p>Click the link below to authenticate:</p><p><a href="{auth_url}" target="_blank">{auth_url}</a></p><p>After authentication, return to the app.</p>'
        else:
            return "<h1>❌ Spotify Authentication Failed</h1><p>Could not initialize Spotify client. Check your credentials.</p>", 400
    except Exception as e:
        print(f"🔴 Error starting Spotify auth: {e}")
        return f"<h1>❌ Spotify Authentication Error</h1><p>{str(e)}</p>", 500

@app.route('/auth/tidal')
def auth_tidal():
    """
    Initiates Tidal OAuth authentication flow
    """
    print("🔐🔐🔐 TIDAL AUTH ROUTE CALLED 🔐🔐🔐")
    try:
        # Create a fresh tidal client to get OAuth URL
        from core.tidal_client import TidalClient
        temp_tidal_client = TidalClient()
        
        if not temp_tidal_client.client_id:
            return "<h1>❌ Tidal Authentication Failed</h1><p>Tidal client ID not configured. Check your credentials.</p>", 400
        
        # Generate PKCE challenge and store globally
        temp_tidal_client._generate_pkce_challenge()
        
        # Store PKCE values globally for callback use
        global tidal_oauth_state
        with tidal_oauth_lock:
            tidal_oauth_state["code_verifier"] = temp_tidal_client.code_verifier
            tidal_oauth_state["code_challenge"] = temp_tidal_client.code_challenge
        
        print(f"🔐 Stored PKCE - verifier: {temp_tidal_client.code_verifier[:20]}... challenge: {temp_tidal_client.code_challenge[:20]}...")
        
        # Create OAuth URL
        import urllib.parse
        params = {
            'response_type': 'code',
            'client_id': temp_tidal_client.client_id,
            'redirect_uri': temp_tidal_client.redirect_uri,
            'scope': 'user.read playlists.read',
            'code_challenge': temp_tidal_client.code_challenge,
            'code_challenge_method': 'S256'
        }
        
        auth_url = f"{temp_tidal_client.auth_url}?" + urllib.parse.urlencode(params)
        
        print(f"🔗 Generated Tidal OAuth URL: {auth_url}")
        print(f"🔗 Redirect URI in URL: {params['redirect_uri']}")
        
        add_activity_item("🔐", "Tidal Auth Started", "Please complete OAuth in browser", "Now")

        # Detect if accessing remotely (copied from Spotify auth logic)
        host = request.host.split(':')[0]
        is_remote = host not in ['127.0.0.1', 'localhost']
        is_docker = os.path.exists('/.dockerenv')
        
        # If in Docker and accessing via 127.0.0.1, recommend localhost
        if is_docker and host == '127.0.0.1':
            host = 'localhost'

        if is_remote or is_docker:
            # Show instructions for remote/docker access
            page_title = "🔐 Tidal Authentication (Remote/Docker)"
            step_1_text = "Click the link below to authenticate with Tidal"
            
            return f'''
            <html>
            <head>
                <style>
                    body {{ font-family: Arial, sans-serif; padding: 20px; max-width: 800px; margin: 0 auto; }}
                    code {{ background: #f0f0f0; padding: 10px; display: block; margin: 10px 0; }}
                    .highlight {{ background: #e8f5e9; }}
                    .copy-btn {{
                        background: #000000; /* Tidal Black */
                        color: white;
                        border: none;
                        padding: 8px 16px;
                        cursor: pointer;
                        border-radius: 4px;
                        font-size: 14px;
                        margin-left: 10px;
                    }}
                    .copy-btn:hover {{ background: #333333; }}
                    .copied {{ background: #4CAF50 !important; }}
                </style>
            </head>
            <body>
                <h1>{page_title}</h1>
                <p><strong>Step 1:</strong> {step_1_text}</p>
                <p><a href="{auth_url}" target="_blank" style="font-size: 18px; color: #000000;">{auth_url}</a></p>
                <hr>
                <p><strong>Step 2:</strong> After authorizing, you'll see a blank page or an error. The URL will look like:</p>
                <code>http://127.0.0.1:8888/tidal/callback?code=...</code>
                <p><strong>Step 3:</strong> Change <code style="display: inline; background: #ffe6e6; padding: 2px 6px;">127.0.0.1</code> to <code style="display: inline; background: #e8f5e9; padding: 2px 6px;">{host}</code> and press Enter:
                    <button class="copy-btn" onclick="copyIP()">Copy IP</button>
                </p>
                <code class="highlight">http://{host}:8888/tidal/callback?code=...</code>
                <p>Authentication will then complete!</p>

                <script>
                    function copyIP() {{
                        navigator.clipboard.writeText('{host}').then(() => {{
                            const btn = event.target;
                            btn.textContent = '✓ Copied!';
                            btn.classList.add('copied');
                            setTimeout(() => {{
                                btn.textContent = 'Copy IP';
                                btn.classList.remove('copied');
                            }}, 2000);
                        }});
                    }}
                </script>
            </body>
            </html>
            '''
        else:
            return f'<h1>🔐 Tidal Authentication</h1><p>Please visit this URL to authenticate:</p><p><a href="{auth_url}" target="_blank">{auth_url}</a></p><p>After authentication, return to the app.</p>'
        
    except Exception as e:
        print(f"🔴 Error starting Tidal auth: {e}")
        import traceback
        print(f"🔴 Full traceback: {traceback.format_exc()}")
        return f"<h1>❌ Tidal Authentication Error</h1><p>{str(e)}</p>", 500


@app.route('/callback')
def spotify_callback():
    """
    Handles Spotify OAuth callback via the main Flask app (port 8008).
    This is the recommended callback for reverse proxy / Docker setups.
    The dedicated HTTPServer on port 8888 continues to work for direct/local access.
    """
    global spotify_client
    auth_code = request.args.get('code')

    if not auth_code:
        error = request.args.get('error')
        if error:
            print(f"🔴 Spotify OAuth error on port 8008: Spotify returned error: {error}")
            add_activity_item("❌", "Spotify Auth Failed", f"Spotify returned error: {error}", "Now")
            return f"<h1>Spotify Authentication Failed</h1><p>Spotify returned error: {error}</p>", 400

        # No code AND no error — check if query params were stripped
        if request.args:
            print(f"🔴 Spotify callback on port 8008 received unexpected params: {dict(request.args)}")
        else:
            # Completely empty — likely a healthcheck or spurious request
            pass
        return '', 204

    print(f"🎵 Spotify callback received on port 8008 with authorization code")

    try:
        from core.spotify_client import SpotifyClient
        from spotipy.oauth2 import SpotifyOAuth
        from config.settings import config_manager

        config = config_manager.get_spotify_config()
        configured_uri = config.get('redirect_uri', "http://127.0.0.1:8888/callback")
        print(f"🎵 Using redirect_uri for token exchange: {configured_uri}")

        auth_manager = SpotifyOAuth(
            client_id=config['client_id'],
            client_secret=config['client_secret'],
            redirect_uri=configured_uri,
            scope="user-library-read user-read-private playlist-read-private playlist-read-collaborative user-read-email",
            cache_path='config/.spotify_cache'
        )

        token_info = auth_manager.get_access_token(auth_code, as_dict=True)

        if token_info:
            spotify_client = SpotifyClient()
            if spotify_client.is_authenticated():
                # Invalidate status cache so next poll picks up the new connection
                _status_cache_timestamps['spotify'] = 0
                # Refresh enrichment worker's client so it picks up new auth
                if spotify_enrichment_worker and hasattr(spotify_enrichment_worker, 'client'):
                    spotify_enrichment_worker.client.reload_config()
                add_activity_item("✅", "Spotify Auth Complete", "Successfully authenticated with Spotify", "Now")
                return "<h1>Spotify Authentication Successful!</h1><p>You can close this window.</p>"
            else:
                raise Exception("Token exchange succeeded but authentication validation failed")
        else:
            raise Exception("Failed to exchange authorization code for access token")
    except Exception as e:
        print(f"🔴 Spotify OAuth callback error on port 8008: {e}")
        add_activity_item("❌", "Spotify Auth Failed", f"Token processing failed: {str(e)}", "Now")
        return f"<h1>Spotify Authentication Failed</h1><p>{str(e)}</p>", 400


@app.route('/api/spotify/disconnect', methods=['POST'])
def spotify_disconnect():
    """Disconnect Spotify and fall back to iTunes/Apple Music"""
    global spotify_client
    try:
        spotify_client.disconnect()
        # Immediately update status cache so UI reflects the change
        _status_cache['spotify'] = {
            'connected': True,  # iTunes fallback is always available
            'response_time': 0,
            'source': 'itunes'
        }
        _status_cache_timestamps['spotify'] = time.time()
        add_activity_item("🔌", "Spotify Disconnected", "Switched to Apple Music/iTunes metadata source", "Now")
        return jsonify({'success': True, 'message': 'Spotify disconnected. Now using Apple Music/iTunes.'})
    except Exception as e:
        logger.error(f"Error disconnecting Spotify: {e}")
        return jsonify({'success': False, 'error': str(e)}), 500


@app.route('/tidal/callback')
def tidal_callback():
    """
    Handles the callback from Tidal after the user authorizes the application.
    It receives an authorization code, exchanges it for an access token,
    and saves the token.
    """
    global tidal_client # We will re-initialize the global client
    auth_code = request.args.get('code')
    
    if not auth_code:
        error = request.args.get('error', 'Unknown error')
        error_description = request.args.get('error_description', 'No description provided.')
        return f"<h1>Tidal Authentication Failed</h1><p>Error: {error}</p><p>{error_description}</p><p>Please close this window and try again.</p>", 400

    try:
        # Create a temporary client for the token exchange
        temp_tidal_client = TidalClient()
        success = temp_tidal_client.fetch_token_from_code(auth_code)
        
        if success:
            # Re-initialize the main global tidal_client instance with the new token
            tidal_client = TidalClient()
            return "<h1>✅ Tidal Authentication Successful!</h1><p>You can now close this window and return to the SoulSync application.</p>"
        else:
            return "<h1>❌ Tidal Authentication Failed</h1><p>Could not exchange authorization code for a token. Please try again.</p>", 400
    except Exception as e:
        print(f"🔴 Error during Tidal token exchange: {e}")
        return f"<h1>❌ An Error Occurred</h1><p>An unexpected error occurred during the authentication process: {e}</p>", 500


# --- Beatport Data API ---

@app.route('/api/beatport/hero-tracks')
def get_beatport_hero_tracks():
    """Get fresh tracks from Beatport hero slideshow for the rebuild slider"""
    try:
        logger.info("🎯 Fetching Beatport hero tracks...")

        # Check cache first
        cached_data = get_cached_beatport_data('homepage', 'hero_tracks')
        if cached_data:
            logger.info("🎯 Returning cached hero tracks data")
            response = jsonify(cached_data)
            return add_cache_headers(response, 3600)  # 1 hour

        # Cache miss - scrape fresh data
        logger.info("🔄 Cache miss - scraping fresh hero tracks data...")

        # Initialize scraper
        scraper = BeatportUnifiedScraper()

        # Get tracks from hero slideshow (increased limit to capture all slides)
        tracks = scraper.scrape_new_on_beatport_hero(limit=15)

        # SMART FILTERING - Remove duplicates and invalid tracks
        valid_tracks = []
        seen_urls = set()

        logger.info(f"🔍 Processing {len(tracks)} raw tracks from scraper (SMART FILTERING)...")

        for i, track in enumerate(tracks):
            logger.info(f"   Track {i+1}: {track.get('title', 'NO_TITLE')} - {track.get('artist', 'NO_ARTIST')}")
            logger.info(f"      URL: {track.get('url', 'NO_URL')}")
            logger.info(f"      Image: {'YES' if track.get('image_url') else 'NO'}")

            # Extract and clean basic data
            title = track.get('title', '').strip()
            artist = track.get('artist', '').strip()
            url = track.get('url', '').strip()
            image_url = track.get('image_url', '').strip()

            # Apply text cleaning for proper spacing
            if title:
                title = clean_beatport_text(title)
            if artist:
                artist = clean_beatport_text(artist)

            # Validation filters
            is_valid = True
            skip_reasons = []

            # Filter 1: Must have title (artist can be fallback)
            if not title or title in ['No title', 'MISSING', 'Unknown Title']:
                is_valid = False
                skip_reasons.append("Missing/invalid title")

            # If no artist, use fallback based on URL or default
            if not artist or artist in ['No artist', 'MISSING', 'Unknown Artist', 'NO_ARTIST']:
                if url and '/release/' in url:
                    artist = 'Various Artists'  # Release pages often have multiple artists
                else:
                    artist = 'Unknown Artist'

            # Filter 2: Must have valid URL and image
            if not url or not image_url:
                is_valid = False
                skip_reasons.append("Missing URL or image")

            # Filter 3: URL must be a track/release page (not promotional pages)
            if url and not any(pattern in url for pattern in ['/release/', '/track/']):
                is_valid = False
                skip_reasons.append("URL is not a track/release page")

            # Filter 4: Deduplication by URL (most reliable method)
            if url in seen_urls:
                is_valid = False
                skip_reasons.append("Duplicate URL")

            if not is_valid:
                logger.info(f"      ❌ Track {i+1} filtered out: {', '.join(skip_reasons)}")
                continue

            # Mark URL as seen for deduplication
            seen_urls.add(url)

            # Clean up title
            title = title.replace(" t ", "'t ").replace("(Extended)DJ", "(Extended)")

            # Clean up artist names
            if 'SyrossianHappy' in artist:
                artist = 'Darius Syrossian'
            if 'Carroll,' in artist:
                artist = 'Ron Carroll'
            if artist.endswith('DJ') and ' ' not in artist[-4:]:
                artist = artist[:-2].strip()

            # Create clean track data
            track_data = {
                'title': title,
                'artist': artist,
                'url': url,
                'image_url': image_url,
                'genre': 'Electronic',  # Default genre
                'year': datetime.now().year
            }

            # Determine genre based on artist
            genre_mapping = {
                'thakzin': 'Afro House',
                'yaya': 'Tech House',
                'darius syrossian': 'Techno',
                'ron carroll': 'House',
                'dj minx': 'House',
                'durante': 'Progressive House'
            }

            for artist_key, mapped_genre in genre_mapping.items():
                if artist_key in artist.lower():
                    track_data['genre'] = mapped_genre
                    break

            valid_tracks.append(track_data)
            logger.info(f"      ✅ Track {i+1} added: {title} - {artist}")

        logger.info(f"✅ Retrieved {len(valid_tracks)} valid unique Beatport tracks (SMART FILTERING)")

        # Prepare response data
        response_data = {
            'success': True,
            'tracks': valid_tracks,
            'count': len(valid_tracks),
            'timestamp': datetime.now().isoformat()
        }

        # Cache the successful response
        set_cached_beatport_data('homepage', 'hero_tracks', response_data)

        response = jsonify(response_data)
        return add_cache_headers(response, 3600)  # 1 hour

    except Exception as e:
        logger.error(f"❌ Error fetching Beatport tracks: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e),
            'tracks': []
        }), 500

@app.route('/api/beatport/new-releases')
def get_beatport_new_releases():
    """Get new releases from Beatport for the rebuild slider grid"""
    try:
        logger.info("🆕 Fetching Beatport new releases...")

        # Check cache first
        cached_data = get_cached_beatport_data('homepage', 'new_releases')
        if cached_data:
            logger.info("🆕 Returning cached new releases data")
            response = jsonify(cached_data)
            return add_cache_headers(response, 3600)  # 1 hour

        # Cache miss - scrape fresh data
        logger.info("🔄 Cache miss - scraping fresh new releases data...")

        # Initialize scraper
        scraper = BeatportUnifiedScraper()

        # Get page and extract releases
        soup = scraper.get_page(scraper.base_url)
        if not soup:
            raise Exception("Could not fetch Beatport homepage")

        # Extract release cards using the working CSS selector
        release_cards = soup.select('.ReleaseCard-style__Wrapper-sc-7c61989b-12.duhBUN')
        releases = []

        logger.info(f"🔍 Found {len(release_cards)} release cards")

        for i, card in enumerate(release_cards[:100]):  # Limit to 100 for 10 slides
            release_data = {}

            # Extract title
            title_elem = card.select_one('[class*="title"], [class*="Title"], h1, h2, h3, h4, h5, h6')
            if title_elem:
                title_text = title_elem.get_text(strip=True)
                if title_text and len(title_text) > 2 and title_text not in ['New Releases', 'Buy', 'Play']:
                    release_data['title'] = title_text

            # Extract artist
            artist_elem = card.select_one('[class*="artist"], [class*="Artist"], a[href*="/artist/"]')
            if artist_elem:
                artist_text = artist_elem.get_text(strip=True)
                if artist_text and len(artist_text) > 1:
                    release_data['artist'] = artist_text

            # Extract label
            label_elem = card.select_one('[class*="label"], [class*="Label"], a[href*="/label/"]')
            if label_elem:
                label_text = label_elem.get_text(strip=True)
                if label_text and len(label_text) > 1:
                    release_data['label'] = label_text

            # Extract URL
            url_link = card.select_one('a[href*="/release/"]')
            if url_link:
                href = url_link.get('href')
                if href:
                    release_data['url'] = urljoin(scraper.base_url, href)

            # Extract image
            img = card.select_one('img')
            if img:
                src = img.get('src') or img.get('data-src') or img.get('data-lazy-src')
                if src:
                    release_data['image_url'] = src

            # URL fallback for title
            if not release_data.get('title') and release_data.get('url'):
                url_parts = release_data['url'].split('/release/')
                if len(url_parts) > 1:
                    slug = url_parts[1].split('/')[0]
                    release_data['title'] = slug.replace('-', ' ').title()

            # Only add if we have essential data
            if release_data.get('title') and release_data.get('url'):
                # Add fallbacks for missing data
                if not release_data.get('artist'):
                    release_data['artist'] = 'Various Artists'
                if not release_data.get('label'):
                    release_data['label'] = 'Unknown Label'

                releases.append(release_data)

        logger.info(f"✅ Successfully extracted {len(releases)} new releases")

        # Prepare response data
        response_data = {
            'success': True,
            'releases': releases,
            'count': len(releases),
            'slides': (len(releases) + 9) // 10,  # Calculate number of slides needed
            'timestamp': datetime.now().isoformat()
        }

        # Cache the successful response
        set_cached_beatport_data('homepage', 'new_releases', response_data)

        response = jsonify(response_data)
        return add_cache_headers(response, 3600)  # 1 hour

    except Exception as e:
        logger.error(f"❌ Error fetching new releases: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e),
            'releases': []
        }), 500


@app.route('/api/beatport/featured-charts')
def get_beatport_featured_charts():
    """Get featured charts from Beatport for the charts slider grid using GridSlider approach"""
    try:
        logger.info("🔥 Fetching Beatport featured charts...")

        # Check cache first
        cached_data = get_cached_beatport_data('homepage', 'featured_charts')
        if cached_data:
            logger.info("🔥 Returning cached featured charts data")
            response = jsonify(cached_data)
            return add_cache_headers(response, 3600)  # 1 hour

        # Cache miss - scrape fresh data
        logger.info("🔄 Cache miss - scraping fresh featured charts data...")

        # Initialize scraper
        scraper = BeatportUnifiedScraper()

        # Get page and extract charts
        soup = scraper.get_page(scraper.base_url)
        if not soup:
            raise Exception("Could not fetch Beatport homepage")

        # Find Featured Charts GridSlider container (like New Releases)
        gridsliders = soup.select('[class*="GridSlider-style__Wrapper"]')
        featured_container = None

        logger.info(f"🔍 Checking {len(gridsliders)} GridSlider containers for featured charts...")

        for container in gridsliders:
            h2 = container.select_one('h2')
            if h2:
                title = h2.get_text(strip=True).lower()
                logger.info(f"📋 Found section: '{h2.get_text(strip=True)}'")

                if 'featured' in title and 'chart' in title:
                    featured_container = container
                    logger.info(f"🔥 FOUND FEATURED CHARTS: '{h2.get_text(strip=True)}'")
                    break

        if not featured_container:
            logger.warning("❌ No Featured Charts GridSlider container found")
            return jsonify({
                'success': False,
                'error': 'Featured Charts section not found',
                'charts': []
            })

        # Extract charts from the container using chart links
        charts = []
        chart_links = featured_container.select('a[href*="/chart/"]')

        logger.info(f"📊 Found {len(chart_links)} chart links in Featured Charts section")

        for i, link in enumerate(chart_links[:100]):  # Limit to 100 for 10 slides
            chart_data = {}

            # Extract chart name from link text or nearby elements
            name_elem = link.select_one('h3, h4, h5, h6, [class*="title"], [class*="Title"], [class*="name"], [class*="Name"]')
            if name_elem:
                name_text = name_elem.get_text(strip=True)
            else:
                name_text = link.get_text(strip=True)

            if name_text and len(name_text) > 2 and name_text.lower() not in ['featured charts', 'buy', 'play']:
                chart_data['name'] = name_text

                # Extract creator using the specific CSS class pattern from chart cards
                creator = 'Beatport'  # Default

                # Look for the ChartCard Name class that contains the creator
                creator_elem = link.select_one('[class*="ChartCard-style__Name"]')
                if creator_elem:
                    creator_text = creator_elem.get_text(strip=True)
                    if creator_text and len(creator_text) > 1 and creator_text.lower() not in ['by', 'chart', 'featured', 'beatport']:
                        creator = creator_text
                    elif creator_text.lower() == 'beatport':
                        creator = 'Beatport'
                else:
                    # Fallback: look for other creator indicators
                    parent = link.parent
                    if parent:
                        fallback_selectors = [
                            '[class*="artist"]', '[class*="Artist"]',
                            '[class*="creator"]', '[class*="Creator"]',
                            '[class*="author"]', '[class*="Author"]'
                        ]

                        for selector in fallback_selectors:
                            fallback_elem = parent.select_one(selector)
                            if fallback_elem:
                                fallback_text = fallback_elem.get_text(strip=True)
                                if fallback_text and len(fallback_text) > 1 and fallback_text.lower() not in ['by', 'chart', 'featured']:
                                    creator = fallback_text
                                    break

                chart_data['creator'] = creator

                # Extract URL
                href = link.get('href', '')
                if href:
                    if href.startswith('/'):
                        chart_data['url'] = f"https://www.beatport.com{href}"
                    else:
                        chart_data['url'] = href

                # Extract image
                img_elem = link.select_one('img') or (link.parent.select_one('img') if link.parent else None)
                if img_elem:
                    src = img_elem.get('src', '') or img_elem.get('data-src', '')
                    if src:
                        if src.startswith('//'):
                            src = f"https:{src}"
                        elif src.startswith('/'):
                            src = f"https://www.beatport.com{src}"
                        chart_data['image'] = src

                # Only add if we have meaningful data
                if 'name' in chart_data and 'url' in chart_data:
                    charts.append(chart_data)
                    logger.info(f"✅ Chart {len(charts)}: {chart_data['name']} by {chart_data['creator']}")

        logger.info(f"📊 Successfully extracted {len(charts)} featured charts")

        # Prepare response data
        response_data = {
            'success': True,
            'charts': charts,
            'count': len(charts),
            'slides': (len(charts) + 9) // 10,  # Calculate number of slides needed
            'timestamp': datetime.now().isoformat()
        }

        # Cache the successful response
        set_cached_beatport_data('homepage', 'featured_charts', response_data)

        response = jsonify(response_data)
        return add_cache_headers(response, 3600)  # 1 hour

    except Exception as e:
        logger.error(f"❌ Error fetching featured charts: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e),
            'charts': []
        }), 500


@app.route('/api/beatport/dj-charts')
def get_beatport_dj_charts():
    """Get DJ charts from Beatport for the DJ charts slider using Carousel approach"""
    try:
        logger.info("🎧 Fetching Beatport DJ charts...")

        # Check cache first
        cached_data = get_cached_beatport_data('homepage', 'dj_charts')
        if cached_data:
            logger.info("🎧 Returning cached DJ charts data")
            response = jsonify(cached_data)
            return add_cache_headers(response, 3600)  # 1 hour

        # Cache miss - scrape fresh data
        logger.info("🔄 Cache miss - scraping fresh DJ charts data...")

        # Initialize scraper
        scraper = BeatportUnifiedScraper()

        # Get page and extract charts
        soup = scraper.get_page(scraper.base_url)
        if not soup:
            raise Exception("Could not fetch Beatport homepage")

        # Find all Carousel containers
        carousels = soup.select('[class*="Carousel-style__Wrapper"]')
        dj_container = None

        logger.info(f"🔍 Checking {len(carousels)} Carousel containers for DJ charts...")

        # Based on test results, DJ charts are in the second carousel (index 1) with ~9 chart links
        for i, container in enumerate(carousels):
            chart_links = container.select('a[href*="/chart/"]')
            logger.info(f"📋 Carousel {i+1}: {len(chart_links)} chart links")

            # DJ charts container typically has 8-12 chart links (not 99+ like featured charts)
            if 5 <= len(chart_links) <= 15:
                dj_container = container
                logger.info(f"🔥 FOUND DJ CHARTS: Carousel {i+1} with {len(chart_links)} charts")
                break

        if not dj_container:
            logger.warning("❌ No DJ Charts Carousel container found")
            return jsonify({
                'success': False,
                'error': 'DJ Charts section not found',
                'charts': []
            })

        # Extract charts from the container using chart links
        charts = []
        chart_links = dj_container.select('a[href*="/chart/"]')

        logger.info(f"📊 Found {len(chart_links)} DJ chart links")

        for i, link in enumerate(chart_links):
            chart_data = {}

            # Extract chart name from link text or nearby elements
            name_elem = link.select_one('h3, h4, h5, h6, [class*="title"], [class*="Title"], [class*="name"], [class*="Name"]')
            if name_elem:
                name_text = name_elem.get_text(strip=True)
            else:
                name_text = link.get_text(strip=True)

            if name_text and len(name_text) > 2:
                chart_data['name'] = name_text

                # Extract creator - for DJ charts, the chart name might be the artist name
                creator = name_text  # Use chart name as creator for DJ charts

                # Look for additional creator info in parent elements
                parent = link.parent
                if parent:
                    creator_selectors = [
                        '[class*="artist"]', '[class*="Artist"]',
                        '[class*="creator"]', '[class*="Creator"]',
                        '[class*="author"]', '[class*="Author"]'
                    ]

                    for selector in creator_selectors:
                        creator_elem = parent.select_one(selector)
                        if creator_elem:
                            creator_text = creator_elem.get_text(strip=True)
                            if creator_text and len(creator_text) > 1 and creator_text != name_text:
                                creator = creator_text
                                break

                chart_data['creator'] = creator

                # Extract URL
                href = link.get('href', '')
                if href:
                    if href.startswith('/'):
                        chart_data['url'] = f"https://www.beatport.com{href}"
                    else:
                        chart_data['url'] = href

                # Extract image
                img_elem = link.select_one('img') or (link.parent.select_one('img') if link.parent else None)
                if img_elem:
                    src = img_elem.get('src', '') or img_elem.get('data-src', '')
                    if src:
                        if src.startswith('//'):
                            src = f"https:{src}"
                        elif src.startswith('/'):
                            src = f"https://www.beatport.com{src}"
                        chart_data['image'] = src

                # Only add if we have meaningful data
                if 'name' in chart_data and 'url' in chart_data:
                    charts.append(chart_data)
                    logger.info(f"✅ DJ Chart {len(charts)}: {chart_data['name']} by {chart_data['creator']}")

        logger.info(f"📊 Successfully extracted {len(charts)} DJ charts")

        # Prepare response data
        response_data = {
            'success': True,
            'charts': charts,
            'count': len(charts),
            'slides': max(1, (len(charts) + 2) // 3),  # 3 cards per slide
            'timestamp': datetime.now().isoformat()
        }

        # Cache the successful response
        set_cached_beatport_data('homepage', 'dj_charts', response_data)

        response = jsonify(response_data)
        return add_cache_headers(response, 3600)  # 1 hour

    except Exception as e:
        logger.error(f"❌ Error fetching DJ charts: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e),
            'charts': []
        }), 500

# --- Placeholder API Endpoints for Other Pages ---

@app.route('/api/activity')
def get_activity():
    # Placeholder: returns mock activity data
    mock_activity = [
        {"time": "1 min ago", "text": "Service status checked."},
        {"time": "5 min ago", "text": "Application server started."}
    ]
    return jsonify({"activities": mock_activity})

@app.route('/api/playlists')
def get_playlists():
    # Placeholder: returns mock playlist data
    if spotify_client and spotify_client.is_authenticated():
        # In a real implementation, you would call spotify_client.get_user_playlists()
        mock_playlists = [
            {"id": "1", "name": "Chill Vibes"},
            {"id": "2", "name": "Workout Mix"},
            {"id": "3", "name": "Liked Songs"}
        ]
        return jsonify({"playlists": mock_playlists})
    return jsonify({"playlists": [], "error": "Spotify not authenticated."})

@app.route('/api/sync', methods=['POST'])
def start_sync():
    # Placeholder: simulates starting a sync
    return jsonify({"success": True, "message": "Sync process started."})

@app.route('/api/search', methods=['POST'])
def search_music():
    """Real search using soulseek_client"""
    data = request.get_json()
    query = data.get('query')
    if not query:
        return jsonify({"error": "No search query provided."}), 400

    logger.info(f"Web UI Search initiated for: '{query}'")

    # Add activity for search start
    add_activity_item("🔍", "Search Started", f"'{query}'", "Now")
    
    try:
        tracks, albums = run_async(soulseek_client.search(query))

        # Convert to dictionaries for JSON response
        processed_albums = []
        for album in albums:
            album_dict = album.__dict__.copy()
            album_dict["tracks"] = [track.__dict__ for track in album.tracks]
            album_dict["result_type"] = "album"
            processed_albums.append(album_dict)

        processed_tracks = []
        for track in tracks:
            track_dict = track.__dict__.copy()
            track_dict["result_type"] = "track"
            processed_tracks.append(track_dict)
        
        # Sort by quality score
        all_results = sorted(processed_albums + processed_tracks, key=lambda x: x.get('quality_score', 0), reverse=True)

        # Add activity for search completion
        total_results = len(all_results)
        add_activity_item("✅", "Search Complete", f"'{query}' - {total_results} results", "Now")

        return jsonify({"results": all_results})
        
    except Exception as e:
        print(f"Search error: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/enhanced-search', methods=['POST'])
def enhanced_search():
    """
    Unified search across Spotify and local database for enhanced search mode.
    Returns categorized results: DB artists, Spotify artists, albums, and tracks.
    """
    data = request.get_json()
    query = data.get('query', '').strip()

    if not query:
        return jsonify({
            "db_artists": [],
            "spotify_artists": [],
            "spotify_albums": [],
            "spotify_tracks": []
        })

    logger.info(f"Enhanced search initiated for: '{query}'")

    try:
        # Search local database for artists (always)
        database = get_database()
        db_artists_objs = database.search_artists(query, limit=5)

        db_artists = []
        for artist in db_artists_objs:
            image_url = None
            if hasattr(artist, 'thumb_url') and artist.thumb_url:
                image_url = fix_artist_image_url(artist.thumb_url)

            db_artists.append({
                "id": artist.id,
                "name": artist.name,
                "image_url": image_url
            })
            logger.debug(f"DB Artist: {artist.name}, thumb_url: {artist.thumb_url if hasattr(artist, 'thumb_url') else None}, fixed_url: {image_url}")

        spotify_artists = []
        spotify_albums = []
        spotify_tracks = []
        metadata_source = "spotify"

        if _is_hydrabase_active():
            # ── Hydrabase is primary metadata source ──
            metadata_source = "hydrabase"
            try:
                artist_objs = hydrabase_client.search_artists(query, limit=10)
                for artist in artist_objs:
                    spotify_artists.append({
                        "id": artist.id,
                        "name": artist.name,
                        "image_url": artist.image_url
                    })

                album_objs = hydrabase_client.search_albums(query, limit=10)
                for album in album_objs:
                    artist_name = ', '.join(album.artists) if album.artists else 'Unknown Artist'
                    spotify_albums.append({
                        "id": album.id,
                        "name": album.name,
                        "artist": artist_name,
                        "image_url": album.image_url,
                        "release_date": album.release_date,
                        "total_tracks": album.total_tracks,
                        "album_type": album.album_type
                    })

                track_objs = hydrabase_client.search_tracks(query, limit=10)
                for track in track_objs:
                    artist_name = ', '.join(track.artists) if track.artists else 'Unknown Artist'
                    spotify_tracks.append({
                        "id": track.id,
                        "name": track.name,
                        "artist": artist_name,
                        "album": track.album,
                        "duration_ms": track.duration_ms,
                        "image_url": track.image_url,
                        "release_date": track.release_date
                    })

                logger.info(f"Hydrabase search results: {len(spotify_artists)} artists, {len(spotify_albums)} albums, {len(spotify_tracks)} tracks")

                # Fire off background comparison with pre-computed Hydrabase counts
                _run_background_comparison(query, hydrabase_counts={
                    'tracks': len(track_objs),
                    'artists': len(artist_objs),
                    'albums': len(album_objs)
                })

            except Exception as e:
                logger.error(f"Hydrabase search failed, falling back to Spotify/iTunes: {e}")
                metadata_source = "spotify"
                spotify_artists = []
                spotify_albums = []
                spotify_tracks = []

        if metadata_source != "hydrabase":
            # ── Standard Spotify/iTunes path ──
            # Mirror to Hydrabase worker (fire-and-forget)
            if hydrabase_worker and dev_mode_enabled:
                hydrabase_worker.enqueue(query, 'track')
                hydrabase_worker.enqueue(query, 'album')
                hydrabase_worker.enqueue(query, 'artists')

            if spotify_client and spotify_client.is_authenticated():
                artist_objs = spotify_client.search_artists(query, limit=10)
                for artist in artist_objs:
                    spotify_artists.append({
                        "id": artist.id,
                        "name": artist.name,
                        "image_url": artist.image_url
                    })

                album_objs = spotify_client.search_albums(query, limit=10)
                for album in album_objs:
                    artist_name = ', '.join(album.artists) if album.artists else 'Unknown Artist'
                    spotify_albums.append({
                        "id": album.id,
                        "name": album.name,
                        "artist": artist_name,
                        "image_url": album.image_url,
                        "release_date": album.release_date,
                        "total_tracks": album.total_tracks,
                        "album_type": album.album_type
                    })

                track_objs = spotify_client.search_tracks(query, limit=10)
                for track in track_objs:
                    artist_name = ', '.join(track.artists) if track.artists else 'Unknown Artist'
                    spotify_tracks.append({
                        "id": track.id,
                        "name": track.name,
                        "artist": artist_name,
                        "album": track.album,
                        "duration_ms": track.duration_ms,
                        "image_url": track.image_url,
                        "release_date": track.release_date
                    })

        logger.info(f"Enhanced search results ({metadata_source}): {len(db_artists)} DB artists, {len(spotify_artists)} artists, {len(spotify_albums)} albums, {len(spotify_tracks)} tracks")

        return jsonify({
            "db_artists": db_artists,
            "spotify_artists": spotify_artists,
            "spotify_albums": spotify_albums,
            "spotify_tracks": spotify_tracks,
            "metadata_source": metadata_source
        })

    except Exception as e:
        logger.error(f"Enhanced search error: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/enhanced-search/stream-track', methods=['POST'])
def stream_enhanced_search_track():
    """
    Quick slskd search for a single track to stream from enhanced search.
    Uses multi-query retry strategy to work around Soulseek keyword filtering.
    Returns the best matching result from Soulseek.
    """
    data = request.get_json()
    track_name = data.get('track_name', '').strip()
    artist_name = data.get('artist_name', '').strip()
    album_name = data.get('album_name', '').strip()
    duration_ms = data.get('duration_ms', 0)

    if not track_name or not artist_name:
        return jsonify({"error": "Track name and artist name are required"}), 400

    logger.info(f"▶️ Enhanced search stream request: '{track_name}' by '{artist_name}'")

    try:
        # Create a temporary SpotifyTrack-like object for the matching engine
        temp_track = type('TempTrack', (), {
            'name': track_name,
            'artists': [artist_name],
            'album': album_name if album_name else None,
            'duration_ms': duration_ms
        })()

        # Generate search queries based on download source mode
        # - Soulseek: Track name only (avoids keyword filtering on artist names)
        # - YouTube: Include artist name (provides context to find actual music)
        download_mode = config_manager.get('download_source.mode', 'soulseek')
        search_queries = []
        import re

        if download_mode in ('youtube', 'tidal') or (download_mode == 'hybrid' and config_manager.get('download_source.hybrid_primary') in ('youtube', 'tidal')):
            # YouTube/Tidal mode: Include artist for better context
            # Primary query: Artist + Track
            if artist_name and track_name:
                search_queries.append(f"{artist_name} {track_name}".strip())

            # Fallback: Artist + Cleaned track (remove parentheses/brackets)
            cleaned_name = re.sub(r'\s*\([^)]*\)', '', track_name).strip()
            cleaned_name = re.sub(r'\s*\[[^\]]*\]', '', cleaned_name).strip()
            if cleaned_name and cleaned_name.lower() != track_name.lower():
                search_queries.append(f"{artist_name} {cleaned_name}".strip())

            logger.info(f"🔍 {download_mode.title()} mode: Searching with artist + track name: {search_queries}")
        else:
            # Soulseek mode: Track name only to avoid keyword filtering
            # Primary query: Full track name
            if track_name.strip():
                search_queries.append(track_name.strip())

            # Cleaned query: Remove parentheses and brackets
            cleaned_name = re.sub(r'\s*\([^)]*\)', '', track_name).strip()
            cleaned_name = re.sub(r'\s*\[[^\]]*\]', '', cleaned_name).strip()

            if cleaned_name and cleaned_name.lower() != track_name.lower():
                search_queries.append(cleaned_name.strip())

            logger.info(f"🔍 Soulseek mode: Searching by track name only (will match with artist): {search_queries}")

        # Remove duplicates while preserving order
        unique_queries = []
        seen = set()
        for query in search_queries:
            if query and query.lower() not in seen:
                unique_queries.append(query)
                seen.add(query.lower())

        search_queries = unique_queries

        # Try queries sequentially until we find a good match
        for query_index, query in enumerate(search_queries):
            logger.info(f"🔍 Query {query_index + 1}/{len(search_queries)}: '{query}'")

            try:
                # Search slskd with timeout
                tracks_result, _ = run_async(soulseek_client.search(query, timeout=15))

                if tracks_result:
                    logger.info(f"✅ Found {len(tracks_result)} results for query: '{query}'")

                    # Use matching engine to find best match
                    best_matches = matching_engine.find_best_slskd_matches_enhanced(temp_track, tracks_result)

                    if best_matches:
                        # Get the first (best) result
                        best_result = best_matches[0]

                        # Convert to dictionary for JSON response (same format as basic search)
                        result_dict = {
                            "username": best_result.username,
                            "filename": best_result.filename,
                            "size": best_result.size,
                            "bitrate": best_result.bitrate,
                            "duration": best_result.duration,
                            "quality": best_result.quality,
                            "free_upload_slots": best_result.free_upload_slots,
                            "upload_speed": best_result.upload_speed,
                            "queue_length": best_result.queue_length,
                            "result_type": "track"
                        }

                        logger.info(f"✅ Returning best match from query '{query}': {best_result.filename} ({best_result.quality})")

                        return jsonify({
                            "success": True,
                            "result": result_dict
                        })
                    else:
                        logger.info(f"⏭️ No suitable matches for query '{query}', trying next query...")
                else:
                    logger.info(f"⏭️ No results for query '{query}', trying next query...")

            except Exception as search_error:
                logger.warning(f"⚠️ Error searching with query '{query}': {search_error}")
                continue

        # If we get here, none of the queries found a suitable match
        logger.warning(f"❌ No suitable matches found after trying {len(search_queries)} queries")
        return jsonify({
            "success": False,
            "error": "No suitable track found on Soulseek after trying multiple search strategies"
        }), 404

    except Exception as e:
        logger.error(f"❌ Error streaming enhanced search track: {e}", exc_info=True)
        return jsonify({"error": str(e)}), 500

@app.route('/api/download', methods=['POST'])
def start_download():
    """Simple download route"""
    data = request.get_json()
    if not data:
        return jsonify({"error": "No download data provided."}), 400
    
    try:
        result_type = data.get('result_type', 'track')
        
        if result_type == 'album':
            tracks = data.get('tracks', [])
            if not tracks:
                return jsonify({"error": "No tracks found in album."}), 400

            started_downloads = 0
            for track_data in tracks:
                try:
                    username = track_data.get('username')
                    filename = track_data.get('filename')
                    file_size = track_data.get('size', 0)

                    download_id = run_async(soulseek_client.download(
                        username,
                        filename,
                        file_size
                    ))
                    if download_id:
                        # Register download for post-processing (simple transfer to /Transfer)
                        context_key = _make_context_key(username, filename)
                        with matched_context_lock:
                            matched_downloads_context[context_key] = {
                                'search_result': {
                                    'username': username,
                                    'filename': filename,
                                    'size': file_size,
                                    'title': track_data.get('title', 'Unknown'),
                                    'artist': track_data.get('artist', 'Unknown'),
                                    'quality': track_data.get('quality', 'Unknown'),
                                    'is_simple_download': True  # Flag for simple processing
                                },
                                'spotify_artist': None,  # No Spotify metadata
                                'spotify_album': None,
                                'track_info': None
                            }
                        started_downloads += 1
                except Exception as e:
                    logger.error(f"Failed to start track download: {e}")
                    continue

            # Add activity for album download start
            album_name = data.get('album_name', 'Unknown Album')
            logger.info(f"📥 Starting simple album download: '{album_name}' with {started_downloads}/{len(tracks)} tracks")
            add_activity_item("📥", "Album Download Started", f"'{album_name}' - {started_downloads} tracks", "Now")

            return jsonify({
                "success": True,
                "message": f"Started {started_downloads} downloads from album"
            })
        
        else:
            # Single track download
            username = data.get('username')
            filename = data.get('filename')
            file_size = data.get('size', 0)

            logger.info(f"📥 Download request - Username: {username}, Filename: {filename[:50]}...")

            if not username or not filename:
                return jsonify({"error": "Missing username or filename."}), 400

            download_id = run_async(soulseek_client.download(username, filename, file_size))
            logger.info(f"📥 Download ID returned: {download_id}")

            if download_id:
                # Register download for post-processing (simple transfer to /Transfer)
                context_key = _make_context_key(username, filename)
                is_streaming_source = username in ('youtube', 'tidal')
                with matched_context_lock:
                    matched_downloads_context[context_key] = {
                        'search_result': {
                            'username': username,
                            'filename': filename,
                            'size': file_size,
                            'title': data.get('title', 'Unknown'),
                            'artist': data.get('artist', 'Unknown'),
                            'quality': data.get('quality', 'Unknown'),
                            'is_simple_download': True  # Flag for simple processing
                        },
                        'spotify_artist': None,  # No Spotify metadata
                        'spotify_album': None,
                        'track_info': None
                    }
                    source_label = username.title() if is_streaming_source else 'Soulseek'
                    logger.info(f"[{source_label}] Registered simple download for post-processing: {context_key}")

                # Extract track name from filename for activity
                track_name = filename.split('/')[-1] if '/' in filename else filename.split('\\')[-1] if '\\' in filename else filename
                logger.info(f"📥 Starting simple track download: '{track_name}'")
                add_activity_item("📥", "Track Download Started", f"'{track_name}'", "Now")
                return jsonify({"success": True, "message": "Download started"})
            else:
                logger.error(f"Failed to start download for: {filename}")
                return jsonify({"error": "Failed to start download"}), 500
                
    except Exception as e:
        logger.error(f"Download error: {e}")
        return jsonify({"error": str(e)}), 500











def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None):
    """
    Robustly finds a completed file on disk, accounting for name variations and
    unexpected subdirectories. This version uses the superior normalization logic
    from the GUI's matching_engine.py to ensure consistency.

    First searches in download_dir, then optionally searches in transfer_dir if provided.
    Returns tuple (file_path, location) where location is 'downloads' or 'transfer'.
    """
    import re
    import os
    from difflib import SequenceMatcher
    from unidecode import unidecode

    # YOUTUBE/TIDAL SUPPORT: Handle encoded filename format "id||title"
    # Extract just the title part for file matching
    if '||' in api_filename:
        _, title = api_filename.split('||', 1)
        api_filename = title  # Use just the title for file searching

    def normalize_for_finding(text: str) -> str:
        """A powerful normalization function adapted from matching_engine.py."""
        if not text: return ""
        text = unidecode(text).lower()
        # Replace common separators with spaces to preserve word boundaries
        text = re.sub(r'[._/]', ' ', text)
        # Keep alphanumeric, spaces, and hyphens. Remove brackets/parentheses content.
        text = re.sub(r'[\[\(].*?[\]\)]', '', text)
        text = re.sub(r'[^a-z0-9\s-]', '', text)
        # Consolidate multiple spaces
        return ' '.join(text.split()).strip()

    def _path_matches_api_dirs(file_path):
        """Check if ALL api directory components appear in the file's path."""
        path_parts = set(p.lower() for p in file_path.replace('\\', '/').split('/'))
        return all(d in path_parts for d in api_dir_parts)

    def search_in_directory(search_dir, location_name):
        """Search for the file in a specific directory."""
        best_fuzzy_path = None
        highest_fuzzy_similarity = 0.0
        exact_matches = []

        # Walk through the entire directory
        for root, dirs, files in os.walk(search_dir):
            # Skip quarantine folder — contains known-wrong files from AcoustID verification
            dirs[:] = [d for d in dirs if d != 'ss_quarantine']
            for file in files:
                # Direct basename match
                if os.path.basename(file) == target_basename:
                    file_path = os.path.join(root, file)
                    # Fast path: if path aligns with expected directory structure, return now
                    if api_dir_parts and _path_matches_api_dirs(file_path):
                        print(f"✅ Found path-confirmed match in {location_name}: {file_path}")
                        return file_path, 1.0
                    if not api_dir_parts:
                        # No directory info to disambiguate — return first match (original behavior)
                        print(f"✅ Found exact match in {location_name}: {file_path}")
                        return file_path, 1.0
                    exact_matches.append(file_path)
                    continue

                # Check for slskd dedup suffix (e.g. "Song_639067852665564677.flac")
                # slskd appends _<timestamp> when a file with the same name already exists
                file_stem, file_ext_part = os.path.splitext(file)
                stripped_stem = re.sub(r'_\d{10,}$', '', file_stem)
                if stripped_stem != file_stem and stripped_stem + file_ext_part == target_basename:
                    file_path = os.path.join(root, file)
                    if api_dir_parts and _path_matches_api_dirs(file_path):
                        print(f"✅ Found path-confirmed dedup match in {location_name}: {file_path}")
                        return file_path, 1.0
                    if not api_dir_parts:
                        print(f"✅ Found dedup-suffix match in {location_name}: {file_path}")
                        return file_path, 1.0
                    exact_matches.append(file_path)
                    continue

                # Fuzzy matching for variations
                normalized_file = normalize_for_finding(file)
                similarity = SequenceMatcher(None, normalized_target, normalized_file).ratio()

                if similarity > highest_fuzzy_similarity:
                    highest_fuzzy_similarity = similarity
                    best_fuzzy_path = os.path.join(root, file)

        # Return best exact match (disambiguated by path), or fall back to fuzzy
        if exact_matches:
            if len(exact_matches) == 1:
                print(f"✅ Found exact match in {location_name}: {exact_matches[0]}")
                return exact_matches[0], 1.0
            # Multiple files share the basename — pick the one whose path best
            # matches the expected directory structure from the Soulseek remote path
            best = exact_matches[0]
            best_score = -1
            for m in exact_matches:
                m_parts = set(p.lower() for p in m.replace('\\', '/').split('/'))
                score = sum(1 for d in api_dir_parts if d in m_parts)
                if score > best_score:
                    best_score = score
                    best = m
            print(f"⚠️ Found {len(exact_matches)} files named '{target_basename}' in {location_name}, picked best path match: {best}")
            return best, 1.0

        return best_fuzzy_path, highest_fuzzy_similarity

    # Extract filename using the helper function
    target_basename = extract_filename(api_filename)
    normalized_target = normalize_for_finding(target_basename)

    # Extract directory components from the API path for disambiguation.
    # When multiple downloads produce the same basename (e.g., "01 - Silent Night.flac"
    # from different albums/users), these let us pick the correct file on disk.
    api_path_normalized = api_filename.replace('\\', '/') if api_filename else ''
    api_dir_parts = [p.lower() for p in api_path_normalized.split('/')[:-1] if p]

    # First search in downloads directory
    best_downloads_path, downloads_similarity = search_in_directory(download_dir, 'downloads')

    # Use a high confidence threshold for fuzzy matches to prevent false positives
    if downloads_similarity > 0.85:
        location = 'downloads'
        if downloads_similarity < 1.0:
            print(f"✅ Found fuzzy match in downloads ({downloads_similarity:.2f}): {best_downloads_path}")
        return (best_downloads_path, location)

    # If not found in downloads and transfer_dir is provided, search there
    transfer_similarity = 0.0  # Initialize transfer_similarity
    if transfer_dir and os.path.exists(transfer_dir):
        best_transfer_path, transfer_similarity = search_in_directory(transfer_dir, 'transfer')

        if transfer_similarity > 0.85:
            location = 'transfer'
            if transfer_similarity < 1.0:
                print(f"✅ Found fuzzy match in transfer ({transfer_similarity:.2f}): {best_transfer_path}")
            return (best_transfer_path, location)

    # Don't spam logs - file not found is common for completed/processed downloads
    return (None, None)




@app.route('/api/downloads/status')
def get_download_status():
    """
    A robust status checker that correctly finds completed files by searching
    the entire download directory with fuzzy matching, mirroring the logic from downloads.py.
    """
    if not soulseek_client:
        return jsonify({"transfers": []})

    try:
        global _processed_download_ids
        transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))

        # Don't return early if no Soulseek transfers - YouTube/Tidal downloads need to be checked too!
        all_transfers = []
        completed_matched_downloads = []
        # Track files already claimed this poll cycle to prevent two contexts from
        # grabbing the same physical file when downloads share a basename (e.g.,
        # "07 - Aurora.flac" from two different albums/artists).
        _files_claimed_this_cycle = set()

        if not transfers_data:
            # No Soulseek transfers, but continue to check YouTube/Tidal downloads below
            pass
        else:
            # This logic now correctly processes the nested structure from the slskd API
            for user_data in transfers_data:
                username = user_data.get('username', 'Unknown')
                if 'directories' in user_data:
                    for directory in user_data['directories']:
                        if 'files' in directory:
                            for file_info in directory['files']:
                                file_info['username'] = username
                                all_transfers.append(file_info)
                                state = file_info.get('state', '').lower()

                                # Check for completion state
                                if ('succeeded' in state or 'completed' in state) and 'errored' not in state and 'rejected' not in state:
                                    # Verify bytes actually transferred before trusting state
                                    _fi_size = file_info.get('size', 0)
                                    _fi_transferred = file_info.get('bytesTransferred', 0)
                                    if _fi_size > 0 and _fi_transferred < _fi_size:
                                        continue  # Not truly complete yet
                                    filename_from_api = file_info.get('filename')
                                    if not filename_from_api: continue

                                    # Check if this completed download has a matched context
                                    context_key = _make_context_key(username, filename_from_api)

                                    # Check if this is an orphaned download that completed after retry
                                    if context_key in _orphaned_download_keys:
                                        # Safety check: if a new context exists for this key, the retry
                                        # re-claimed the same source — treat it as active, not orphaned
                                        with matched_context_lock:
                                            has_active_context = context_key in matched_downloads_context
                                        if has_active_context:
                                            print(f"🔄 Orphaned key {context_key} has active context — retry re-used same source, treating as active")
                                            _orphaned_download_keys.discard(context_key)
                                            # Fall through to normal processing below
                                        else:
                                            download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
                                            found_result = _find_completed_file_robust(download_dir, filename_from_api)
                                            found_path = found_result[0] if found_result and found_result[0] else None
                                            orphan_cleaned = False
                                            if found_path:
                                                try:
                                                    os.remove(found_path)
                                                    print(f"🧹 Deleted orphaned download: {os.path.basename(found_path)}")
                                                    orphan_cleaned = True
                                                except Exception as e:
                                                    print(f"⚠️ Failed to delete orphaned file (will retry next poll): {e}")
                                            else:
                                                # File not on disk (already gone or never written) — nothing to clean
                                                orphan_cleaned = True
                                            if orphan_cleaned:
                                                # Remove transfer from slskd
                                                transfer_id = file_info.get('id')
                                                if transfer_id:
                                                    try:
                                                        run_async(soulseek_client.cancel_download(str(transfer_id), username, remove=True))
                                                    except Exception:
                                                        pass
                                                _orphaned_download_keys.discard(context_key)
                                            continue  # Skip normal post-processing either way

                                    # Skip downloads we've already processed (prevents log spam)
                                    if context_key in _processed_download_ids:
                                        continue

                                    with matched_context_lock:
                                        context = matched_downloads_context.get(context_key)
                                        available_keys = list(matched_downloads_context.keys())[:5] if not context else None

                                    if context:
                                        print(f"✅ [Context Lookup] Found context for key: {context_key}")
                                    elif context_key not in _stale_transfer_keys:
                                        # Only log once per stale key to avoid spamming every poll cycle
                                        print(f"⚠️ [Context Lookup] No context found for key: {context_key}")
                                        print(f"   Available keys: {available_keys}...")
                                        _stale_transfer_keys.add(context_key)

                                    if context:
                                        download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
                                        # Use the new robust file finder (only search downloads for post-processing candidates)
                                        found_result = _find_completed_file_robust(download_dir, filename_from_api)
                                        found_path = found_result[0] if found_result and found_result[0] else None

                                        if found_path:
                                            # Prevent two contexts from claiming the same physical file
                                            _norm_path = os.path.normpath(found_path)
                                            if _norm_path in _files_claimed_this_cycle:
                                                print(f"⚠️ File already claimed by another context this cycle: {os.path.basename(found_path)} — deferring to next poll")
                                            else:
                                                _files_claimed_this_cycle.add(_norm_path)
                                                print(f"🎯 Found completed matched file on disk: {found_path}")
                                                completed_matched_downloads.append((context_key, context, found_path))
                                                # Don't add to _processed_download_ids yet - wait until thread starts successfully

                                                # Clean up retry tracking if file was found after retries
                                                with _download_retry_lock:
                                                    if context_key in _download_retry_attempts:
                                                        retry_count = _download_retry_attempts[context_key]['count']
                                                        elapsed = time.time() - _download_retry_attempts[context_key]['first_attempt']
                                                        print(f"✅ File found after {retry_count} retry attempt(s) ({elapsed:.1f}s): {os.path.basename(filename_from_api)}")
                                                        del _download_retry_attempts[context_key]
                                        else:
                                            # File not found yet - implement retry logic instead of immediate give-up
                                            # This fixes race condition where slskd reports completion before file is written to disk
                                            with _download_retry_lock:
                                                if context_key not in _download_retry_attempts:
                                                    # First retry attempt
                                                    _download_retry_attempts[context_key] = {
                                                        'count': 1,
                                                        'first_attempt': time.time()
                                                    }
                                                    print(f"⏳ File not found yet: '{os.path.basename(filename_from_api)}' - Will retry (attempt 1/{_download_retry_max})")
                                                else:
                                                    # Increment retry count
                                                    _download_retry_attempts[context_key]['count'] += 1
                                                    retry_count = _download_retry_attempts[context_key]['count']
                                                    elapsed = time.time() - _download_retry_attempts[context_key]['first_attempt']

                                                    if retry_count >= _download_retry_max:
                                                        # Max retries reached, give up
                                                        print(f"❌ CRITICAL: Could not find '{os.path.basename(filename_from_api)}' after {retry_count} attempts over {elapsed:.1f}s. Giving up.")
                                                        _processed_download_ids.add(context_key)
                                                        # Clean up retry tracking
                                                        del _download_retry_attempts[context_key]
                                                    else:
                                                        print(f"⏳ File not found yet: '{os.path.basename(filename_from_api)}' - Will retry (attempt {retry_count}/{_download_retry_max}, elapsed: {elapsed:.1f}s)")

        # If we found completed matched downloads, start processing them in background threads
        if completed_matched_downloads:
            def process_completed_downloads():
                for context_key, context, found_path in completed_matched_downloads:
                    try:
                        print(f"🚀 Starting post-processing thread for: {context_key}")
                        # Start the post-processing in a separate thread
                        thread = threading.Thread(target=_post_process_matched_download, args=(context_key, context, found_path))
                        thread.daemon = True
                        thread.start()

                        # Only mark as processed AFTER thread starts successfully
                        _processed_download_ids.add(context_key)
                        print(f"✅ Marked as processed: {context_key}")

                        # DON'T remove context immediately - verification worker needs it
                        # Context will be cleaned up by verification worker after both processors complete
                        print(f"💾 Keeping context for verification worker: {context_key}")

                    except Exception as e:
                        print(f"❌ Error starting post-processing thread for {context_key}: {e}")
                        # Don't add to processed set if thread failed to start
                        print(f"⚠️ Will retry {context_key} on next check")

            # Start a single thread to manage the launching of all processing threads
            processing_thread = threading.Thread(target=process_completed_downloads)
            processing_thread.daemon = True
            processing_thread.start()

        # Also include YouTube/Tidal downloads in the response
        try:
            all_streaming_downloads = run_async(soulseek_client.get_all_downloads())

            for download in all_streaming_downloads:
                if download.username in ('youtube', 'tidal'):
                    source_label = download.username.title()
                    # Convert DownloadStatus to transfer format that frontend expects
                    streaming_transfer = {
                        'id': download.id,
                        'filename': download.filename,
                        'username': download.username,
                        'state': download.state,
                        'percentComplete': download.progress,
                        'size': download.size,
                        'bytesTransferred': download.transferred,
                        'averageSpeed': download.speed,
                        'direction': 'Download',  # Required by frontend
                    }
                    all_transfers.append(streaming_transfer)

                    # Check if download is completed and needs post-processing
                    # Verify bytes match before trusting state string
                    _st_bytes_ok = download.size <= 0 or download.transferred >= download.size
                    if _st_bytes_ok and download.state and ('succeeded' in download.state.lower() or 'completed' in download.state.lower()):
                        context_key = _make_context_key(download.username, download.filename)

                        with matched_context_lock:
                            context = matched_downloads_context.get(context_key)

                        if context and context_key not in _processed_download_ids:
                            download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
                            found_result = _find_completed_file_robust(download_dir, download.filename)
                            found_path = found_result[0] if found_result and found_result[0] else None

                            if found_path:
                                # Prevent two contexts from claiming the same physical file
                                _st_norm = os.path.normpath(found_path)
                                if _st_norm in _files_claimed_this_cycle:
                                    print(f"⚠️ [{source_label}] File already claimed this cycle: {os.path.basename(found_path)} — deferring")
                                    continue
                                _files_claimed_this_cycle.add(_st_norm)
                                print(f"🎯 [{source_label}] Found completed matched file on disk: {found_path}")
                                # Start post-processing thread
                                def process_streaming_download(_ctx_key=context_key, _ctx=context, _path=found_path, _label=source_label):
                                    try:
                                        print(f"🚀 [{_label}] Starting post-processing thread for: {_ctx_key}")
                                        thread = threading.Thread(target=_post_process_matched_download, args=(_ctx_key, _ctx, _path))
                                        thread.daemon = True
                                        thread.start()
                                        _processed_download_ids.add(_ctx_key)
                                        print(f"✅ [{_label}] Marked as processed: {_ctx_key}")
                                    except Exception as e:
                                        print(f"❌ [{_label}] Error starting post-processing thread for {_ctx_key}: {e}")

                                processing_thread = threading.Thread(target=process_streaming_download)
                                processing_thread.daemon = True
                                processing_thread.start()
                            else:
                                # File not found - likely already processed and moved to library
                                # Mark as processed to prevent infinite checking
                                _processed_download_ids.add(context_key)
        except Exception as streaming_error:
            import traceback
            print(f"⚠️ Could not fetch YouTube/Tidal downloads for status: {streaming_error}")
            traceback.print_exc()

        return jsonify({"transfers": all_transfers})

    except Exception as e:
        print(f"Error fetching download status: {e}")
        return jsonify({"error": str(e)}), 500





@app.route('/api/downloads/cancel', methods=['POST'])
def cancel_download():
    """
    Cancel a specific download transfer, matching GUI functionality.
    """
    data = request.get_json()
    if not data:
        return jsonify({"success": False, "error": "No data provided."}), 400
    
    download_id = data.get('download_id')
    username = data.get('username')

    if not all([download_id, username]):
        return jsonify({"success": False, "error": "Missing download_id or username."}), 400

    try:
        # Call the same client method the GUI uses
        success = run_async(soulseek_client.cancel_download(download_id, username, remove=True))
        if success:
            return jsonify({"success": True, "message": "Download cancelled."})
        else:
            return jsonify({"success": False, "error": "Failed to cancel download via slskd."}), 500
    except Exception as e:
        print(f"Error cancelling download: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/downloads/cancel-all', methods=['POST'])
def cancel_all_downloads():
    """
    Cancel all active downloads from slskd, then clear completed ones.
    """
    try:
        # First cancel all active downloads
        cancel_success = run_async(soulseek_client.cancel_all_downloads())
        if not cancel_success:
            return jsonify({"success": False, "error": "Failed to cancel active downloads."}), 500

        # Then clear the now-cancelled/completed downloads
        clear_success = run_async(soulseek_client.clear_all_completed_downloads())

        # Sweep empty directories
        _sweep_empty_download_directories()

        return jsonify({"success": True, "message": "All downloads cancelled and cleared."})
    except Exception as e:
        print(f"Error cancelling all downloads: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/downloads/clear-finished', methods=['POST'])
def clear_finished_downloads():
    """
    Clear all terminal (completed, cancelled, failed) downloads from slskd.
    """
    try:
        # This single client call handles clearing everything that is no longer active
        success = run_async(soulseek_client.clear_all_completed_downloads())
        if success:
            # Also sweep empty directories left behind by completed downloads
            _sweep_empty_download_directories()
            return jsonify({"success": True, "message": "Finished downloads cleared."})
        else:
            return jsonify({"success": False, "error": "Backend failed to clear downloads."}), 500
    except Exception as e:
        print(f"Error clearing finished downloads: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/downloads/task/<task_id>/candidates', methods=['GET'])
def get_task_candidates(task_id):
    """Returns the cached search candidates for a download task so the UI can show what was found."""
    try:
        with tasks_lock:
            task = download_tasks.get(task_id)
            if not task:
                return jsonify({"error": "Task not found"}), 404

            candidates = task.get('cached_candidates', [])
            track_info = task.get('track_info', {})
            error_message = task.get('error_message', '')

            serialized = []
            for c in candidates:
                if hasattr(c, '__dict__'):
                    serialized.append({
                        'username': getattr(c, 'username', ''),
                        'filename': getattr(c, 'filename', ''),
                        'size': getattr(c, 'size', 0),
                        'bitrate': getattr(c, 'bitrate', None),
                        'duration': getattr(c, 'duration', None),
                        'quality': getattr(c, 'quality', ''),
                        'free_upload_slots': getattr(c, 'free_upload_slots', 0),
                        'upload_speed': getattr(c, 'upload_speed', 0),
                        'queue_length': getattr(c, 'queue_length', 0),
                        'artist': getattr(c, 'artist', None),
                        'title': getattr(c, 'title', None),
                        'album': getattr(c, 'album', None),
                    })
                elif isinstance(c, dict):
                    serialized.append(c)

        return jsonify({
            "task_id": task_id,
            "track_info": {
                "name": track_info.get('name', 'Unknown') if isinstance(track_info, dict) else 'Unknown',
                "artist": track_info.get('artist', 'Unknown') if isinstance(track_info, dict) else 'Unknown',
            },
            "error_message": error_message,
            "candidates": serialized,
            "candidate_count": len(serialized),
        })
    except Exception as e:
        print(f"❌ [Candidates] Error fetching candidates for task {task_id}: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/downloads/task/<task_id>/download-candidate', methods=['POST'])
def download_selected_candidate(task_id):
    """Restart a not_found/failed task by downloading a user-selected candidate."""
    try:
        data = request.get_json()
        if not data or not data.get('username') or not data.get('filename'):
            return jsonify({"error": "Missing username or filename"}), 400

        username = data['username']
        filename = data['filename']
        size = data.get('size', 0)

        with tasks_lock:
            task = download_tasks.get(task_id)
            if not task:
                return jsonify({"error": "Task not found"}), 404
            if task['status'] not in ('not_found', 'failed'):
                return jsonify({"error": f"Task is {task['status']}, not eligible for retry"}), 400

            batch_id = task.get('batch_id')
            track_info = task.get('track_info', {})

            # Reset task state
            task['status'] = 'downloading'
            task['error_message'] = None
            task['status_change_time'] = time.time()
            task.pop('download_id', None)
            task.pop('username', None)
            task.pop('filename', None)
            # Clear the selected candidate from used_sources so it won't be skipped
            used_sources = task.get('used_sources', set())
            source_key = f"{username}_{filename}"
            used_sources.discard(source_key)

            # Reset batch tracking for this task
            if batch_id and batch_id in download_batches:
                batch = download_batches[batch_id]
                # Remove from completed set so _on_download_completed can fire again
                completed_set = batch.get('_completed_task_ids', set())
                completed_set.discard(task_id)
                # Remove from permanently_failed_tracks
                track_index = task.get('track_index')
                batch['permanently_failed_tracks'] = [
                    t for t in batch.get('permanently_failed_tracks', [])
                    if t.get('table_index') != track_index and t.get('download_index') != track_index
                ]
                # Restore worker slot
                batch['active_count'] = batch.get('active_count', 0) + 1

        # Build a TrackResult-like candidate object
        from core.soulseek_client import TrackResult
        candidate = TrackResult(
            username=username,
            filename=filename,
            size=size,
            bitrate=data.get('bitrate'),
            duration=data.get('duration'),
            quality=data.get('quality', 'unknown'),
            free_upload_slots=data.get('free_upload_slots', 0),
            upload_speed=data.get('upload_speed', 0),
            queue_length=data.get('queue_length', 0),
            artist=data.get('artist'),
            title=data.get('title'),
            album=data.get('album'),
        )
        candidate.confidence = 1.0  # Required by _attempt_download_with_candidates sort

        # Reconstruct Track object from task's track_info
        from core.itunes_client import Track
        artists = track_info.get('artists', [])
        artist_names = []
        for a in (artists if isinstance(artists, list) else []):
            if isinstance(a, dict):
                artist_names.append(a.get('name', 'Unknown'))
            elif isinstance(a, str):
                artist_names.append(a)
        if not artist_names:
            artist_names = [track_info.get('artist', 'Unknown')]

        track = Track(
            id=track_info.get('id', ''),
            name=track_info.get('name', 'Unknown'),
            artists=artist_names,
            album=track_info.get('album', {}).get('name', '') if isinstance(track_info.get('album'), dict) else track_info.get('album', ''),
            duration_ms=track_info.get('duration_ms', 0),
            popularity=0,
        )

        # Submit to thread pool — don't block the request
        def _run_manual_download():
            success = _attempt_download_with_candidates(task_id, [candidate], track, batch_id)
            if not success:
                with tasks_lock:
                    if task_id in download_tasks:
                        download_tasks[task_id]['status'] = 'failed'
                        download_tasks[task_id]['error_message'] = 'Manual download failed to start — user may be offline'
                if batch_id:
                    _on_download_completed(batch_id, task_id, success=False)

        missing_download_executor.submit(_run_manual_download)

        track_name = track_info.get('name', 'Unknown')
        print(f"🎯 [Manual Download] User selected candidate for '{track_name}' from {username}")
        return jsonify({"success": True, "message": f"Download initiated for '{track_name}'"})

    except Exception as e:
        print(f"❌ [Manual Download] Error: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"error": str(e)}), 500

@app.route('/api/quarantine/clear', methods=['POST'])
def clear_quarantine():
    """Delete all files and folders inside the ss_quarantine directory."""
    import shutil
    try:
        download_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
        quarantine_path = os.path.join(download_path, 'ss_quarantine')

        if not os.path.isdir(quarantine_path):
            return jsonify({"success": True, "message": "Quarantine folder is already empty."})

        removed_files = 0
        for entry in os.listdir(quarantine_path):
            entry_path = os.path.join(quarantine_path, entry)
            try:
                if os.path.isfile(entry_path):
                    os.remove(entry_path)
                    removed_files += 1
                elif os.path.isdir(entry_path):
                    shutil.rmtree(entry_path)
                    removed_files += 1
            except Exception as e:
                print(f"⚠️ [Quarantine] Failed to remove {entry}: {e}")

        print(f"🧹 [Quarantine] Cleared {removed_files} item(s) from quarantine folder")
        return jsonify({"success": True, "message": f"Quarantine cleared ({removed_files} item{'s' if removed_files != 1 else ''} removed)."})
    except Exception as e:
        print(f"❌ [Quarantine] Error clearing quarantine: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/scan/request', methods=['POST'])
def request_media_scan():
    """
    Request a media library scan with automatic completion callback support.
    """
    try:
        if not web_scan_manager:
            return jsonify({"success": False, "error": "Scan manager not initialized"}), 500

        data = request.get_json() or {}
        reason = data.get('reason', 'Web UI download completed')
        auto_database_update = data.get('auto_database_update', True)

        def scan_completion_callback():
            """Callback to trigger automatic database update after scan completes"""
            if auto_database_update:
                try:
                    logger.info("🔄 Starting automatic incremental database update after scan completion")
                    # Start database update in a separate thread to avoid blocking
                    threading.Thread(
                        target=trigger_automatic_database_update,
                        args=("Post-scan automatic update",),
                        daemon=True
                    ).start()
                except Exception as e:
                    logger.error(f"Error starting automatic database update: {e}")

        # Request scan with callback
        result = web_scan_manager.request_scan(
            reason=reason,
            callback=scan_completion_callback if auto_database_update else None
        )

        add_activity_item("📡", "Media Scan", f"Scan requested: {reason}", "Now")
        return jsonify({
            "success": True,
            "scan_info": result,
            "auto_database_update": auto_database_update
        })

    except Exception as e:
        logger.error(f"Error requesting media scan: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/scan/status', methods=['GET'])
def get_scan_status():
    """
    Get current media scan status.
    """
    try:
        if not web_scan_manager:
            return jsonify({"success": False, "error": "Scan manager not initialized"}), 500

        status = web_scan_manager.get_scan_status()
        return jsonify({"success": True, "status": status})

    except Exception as e:
        logger.error(f"Error getting scan status: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/database/incremental-update', methods=['POST'])
def request_incremental_database_update():
    """
    Request an incremental database update with prerequisites checking.
    """
    try:
        data = request.get_json() or {}
        reason = data.get('reason', 'Web UI manual request')

        # Check prerequisites (similar to GUI logic)
        db = get_database()

        # Check if database has enough content for incremental updates
        track_count = db.execute("SELECT COUNT(*) FROM tracks").fetchone()[0]
        if track_count < 100:
            return jsonify({
                "success": False,
                "error": f"Database has only {track_count} tracks - insufficient for incremental updates (minimum 100)",
                "track_count": track_count
            }), 400

        # Check if there's been a previous full refresh
        last_refresh = db.execute(
            "SELECT value FROM system_info WHERE key = 'last_full_refresh'"
        ).fetchone()

        if not last_refresh:
            return jsonify({
                "success": False,
                "error": "No previous full refresh found - incremental updates require established database",
                "suggestion": "Run a full refresh first"
            }), 400

        # Start incremental update
        result = trigger_automatic_database_update(reason)

        add_activity_item("🔄", "Database Update", f"Incremental update started: {reason}", "Now")
        return jsonify({
            "success": True,
            "message": "Incremental database update started",
            "track_count": track_count,
            "last_refresh": last_refresh[0] if last_refresh else None,
            "reason": reason
        })

    except Exception as e:
        logger.error(f"Error requesting incremental database update: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

def trigger_automatic_database_update(reason="Automatic update"):
    """
    Helper function to trigger automatic incremental database update.
    """
    try:
        from config.settings import config_manager
        active_server = config_manager.get_active_media_server()

        # Get the appropriate media client
        media_client = None
        if active_server == "jellyfin" and jellyfin_client:
            media_client = jellyfin_client
        elif active_server == "navidrome" and navidrome_client:
            media_client = navidrome_client
        else:
            media_client = plex_client  # Default fallback

        if not media_client or not media_client.is_connected():
            logger.error(f"No connected {active_server} client for automatic database update")
            return False

        # Create and start database update worker
        worker = DatabaseUpdateWorker(
            media_client=media_client,
            server_type=active_server,
            full_refresh=False  # Always incremental for automatic updates
        )

        def update_completion_callback():
            logger.info(f"✅ Automatic incremental database update completed for {active_server}")
            add_activity_item("✅", "Database Update", f"Automatic update completed ({active_server})", "Now")

        # Start update in background thread
        update_thread = threading.Thread(
            target=lambda: worker.run_with_callback(update_completion_callback),
            daemon=True
        )
        update_thread.start()

        logger.info(f"🔄 Started automatic incremental database update for {active_server}")
        return True

    except Exception as e:
        logger.error(f"Error in automatic database update: {e}")
        return False

@app.route('/api/test/automation', methods=['POST'])
def test_automation_workflow():
    """
    Test endpoint to verify the automatic workflow functionality.
    """
    try:
        data = request.get_json() or {}
        test_type = data.get('test_type', 'full')

        results = {}

        # Test 1: Scan manager status
        if web_scan_manager:
            scan_status = web_scan_manager.get_scan_status()
            results['scan_manager'] = {'status': 'available', 'current_status': scan_status}
        else:
            results['scan_manager'] = {'status': 'unavailable'}

        # Test 2: Database prerequisites
        try:
            db = get_database()
            track_count = db.execute("SELECT COUNT(*) FROM tracks").fetchone()[0]
            last_refresh = db.execute(
                "SELECT value FROM system_info WHERE key = 'last_full_refresh'"
            ).fetchone()

            results['database'] = {
                'track_count': track_count,
                'meets_minimum': track_count >= 100,
                'has_previous_refresh': last_refresh is not None,
                'last_refresh': last_refresh[0] if last_refresh else None
            }
        except Exception as e:
            results['database'] = {'error': str(e)}

        # Test 3: Media client connections
        active_server = config_manager.get_active_media_server()
        results['media_clients'] = {'active_server': active_server}

        for client_name, client in [
            ('plex', plex_client),
            ('jellyfin', jellyfin_client),
            ('navidrome', navidrome_client)
        ]:
            try:
                is_connected = client.is_connected() if client else False
                results['media_clients'][client_name] = {
                    'available': client is not None,
                    'connected': is_connected
                }
            except Exception as e:
                results['media_clients'][client_name] = {
                    'available': client is not None,
                    'connected': False,
                    'error': str(e)
                }

        # Test 4: If requested, actually test the scan request
        if test_type == 'full' and web_scan_manager:
            try:
                scan_result = web_scan_manager.request_scan(
                    reason="Automation test",
                    callback=None
                )
                results['scan_test'] = {'success': True, 'result': scan_result}
            except Exception as e:
                results['scan_test'] = {'success': False, 'error': str(e)}

        return jsonify({
            "success": True,
            "test_results": results,
            "automation_ready": (
                results.get('scan_manager', {}).get('status') == 'available' and
                results.get('database', {}).get('meets_minimum', False) and
                results.get('database', {}).get('has_previous_refresh', False)
            )
        })

    except Exception as e:
        logger.error(f"Error in automation test: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/searches/clear-all', methods=['POST'])
def clear_all_searches():
    """
    Clear all searches from slskd search history.
    """
    try:
        success = run_async(soulseek_client.clear_all_searches())
        if success:
            add_activity_item("🧹", "Search Cleanup", "All search history cleared manually", "Now")
            return jsonify({"success": True, "message": "All searches cleared."})
        else:
            return jsonify({"success": False, "error": "Backend failed to clear searches."}), 500
    except Exception as e:
        print(f"Error clearing searches: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/searches/maintain', methods=['POST'])
def maintain_search_history():
    """
    Maintain search history by keeping only recent searches.
    """
    try:
        data = request.get_json() or {}
        keep_searches = data.get('keep_searches', 50)
        trigger_threshold = data.get('trigger_threshold', 200)
        
        success = run_async(soulseek_client.maintain_search_history_with_buffer(
            keep_searches=keep_searches, trigger_threshold=trigger_threshold
        ))
        if success:
            add_activity_item("🧹", "Search Maintenance", f"Search history maintained (keeping {keep_searches} searches)", "Now")
            return jsonify({"success": True, "message": f"Search history maintained (keeping {keep_searches} searches)."})
        else:
            return jsonify({"success": False, "error": "Backend failed to maintain search history."}), 500
    except Exception as e:
        print(f"Error maintaining search history: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

def fix_artist_image_url(thumb_url):
    """Convert localhost URLs to proper server URLs using config"""
    if not thumb_url:
        return None

    try:
        # Check if it's a localhost URL or relative path that needs fixing
        needs_fixing = (
            thumb_url.startswith('http://localhost:') or
            thumb_url.startswith('https://localhost:') or
            thumb_url.startswith('/library/') or  # Plex relative paths
            thumb_url.startswith('/Items/') or    # Jellyfin relative paths
            thumb_url.startswith('/api/') or      # Old Navidrome API paths
            thumb_url.startswith('/rest/')        # Navidrome Subsonic API paths
        )

        if needs_fixing:
            active_server = config_manager.get_active_media_server()
            print(f"🔧 Fixing URL: {thumb_url}, Active server: {active_server}")

            if active_server == 'plex':
                plex_config = config_manager.get_plex_config()
                plex_base_url = plex_config.get('base_url', '')
                plex_token = plex_config.get('token', '')
                print(f"🔧 Plex config - base_url: {plex_base_url}, token: {plex_token[:10]}...")

                if plex_base_url and plex_token:
                    # Extract the path from URL
                    if thumb_url.startswith('/library/'):
                        # Already a path
                        path = thumb_url
                    else:
                        # Full localhost URL, extract path
                        from urllib.parse import urlparse
                        parsed = urlparse(thumb_url)
                        path = parsed.path

                    # Construct proper Plex URL with token
                    fixed_url = f"{plex_base_url.rstrip('/')}{path}?X-Plex-Token={plex_token}"
                    print(f"🔧 Fixed URL: {fixed_url}")
                    return fixed_url

            elif active_server == 'jellyfin':
                jellyfin_config = config_manager.get_jellyfin_config()
                jellyfin_base_url = jellyfin_config.get('base_url', '')
                jellyfin_token = jellyfin_config.get('api_key', '')
                print(f"🔧 Jellyfin config - base_url: {jellyfin_base_url}, token: {jellyfin_token[:10] if jellyfin_token else 'None'}...")

                if jellyfin_base_url:
                    # Extract the path from URL
                    if thumb_url.startswith('/Items/') or thumb_url.startswith('/api/'):
                        # Already a path
                        path = thumb_url
                    else:
                        # Full localhost URL, extract path
                        from urllib.parse import urlparse
                        parsed = urlparse(thumb_url)
                        path = parsed.path

                    # Construct proper Jellyfin URL with token
                    if jellyfin_token:
                        separator = '&' if '?' in path else '?'
                        fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}{separator}X-Emby-Token={jellyfin_token}"
                    else:
                        fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}"
                    print(f"🔧 Fixed URL: {fixed_url}")
                    return fixed_url

            elif active_server == 'navidrome':
                navidrome_config = config_manager.get_navidrome_config()
                navidrome_base_url = navidrome_config.get('base_url', '')
                navidrome_username = navidrome_config.get('username', '')
                navidrome_password = navidrome_config.get('password', '')
                print(f"🔧 Navidrome config - base_url: {navidrome_base_url}, username: {navidrome_username}")

                if navidrome_base_url and navidrome_username and navidrome_password:
                    # Extract the path from URL
                    if thumb_url.startswith('/rest/'):
                        # Already a Subsonic API path
                        path = thumb_url
                    else:
                        # Full localhost URL, extract path
                        from urllib.parse import urlparse
                        parsed = urlparse(thumb_url)
                        path = parsed.path

                    # Generate Subsonic API authentication
                    import hashlib
                    import secrets
                    salt = secrets.token_hex(6)
                    token = hashlib.md5((navidrome_password + salt).encode()).hexdigest()

                    # Add authentication parameters to the URL
                    separator = '&' if '?' in path else '?'
                    auth_params = f"u={navidrome_username}&t={token}&s={salt}&v=1.16.1&c=SoulSync&f=json"

                    # Construct proper Navidrome Subsonic URL
                    fixed_url = f"{navidrome_base_url.rstrip('/')}{path}{separator}{auth_params}"
                    print(f"🔧 Fixed URL: {fixed_url}")
                    return fixed_url

            print(f"⚠️ No configuration found for {active_server} or unsupported server type")

        # Return original URL if no fixing needed/possible
        return thumb_url

    except Exception as e:
        print(f"Error fixing image URL '{thumb_url}': {e}")
        return thumb_url

@app.route('/api/library/artists')
def get_library_artists():
    """Get artists for the library page with search, filtering, and pagination"""
    try:
        # Get query parameters
        search_query = request.args.get('search', '')
        letter = request.args.get('letter', 'all')
        page = int(request.args.get('page', 1))
        limit = int(request.args.get('limit', 75))
        watchlist_filter = request.args.get('watchlist', 'all')

        # Get database instance
        database = get_database()

        # Get artists from database
        result = database.get_library_artists(
            search_query=search_query,
            letter=letter,
            page=page,
            limit=limit,
            watchlist_filter=watchlist_filter,
            profile_id=get_current_profile_id()
        )

        # Fix image URLs for all artists
        for artist in result['artists']:
            if artist.get('image_url'):
                artist['image_url'] = fix_artist_image_url(artist['image_url'])

        return jsonify({
            "success": True,
            **result
        })

    except Exception as e:
        print(f"❌ Error fetching library artists: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({
            "success": False,
            "error": str(e),
            "artists": [],
            "pagination": {
                "page": 1,
                "limit": 75,
                "total_count": 0,
                "total_pages": 0,
                "has_prev": False,
                "has_next": False
            }
        }), 500

@app.route('/api/test-artist/<artist_id>')
def test_artist_endpoint(artist_id):
    """Simple test endpoint"""
    return jsonify({
        "success": True,
        "message": f"Test endpoint working for artist ID: {artist_id}"
    })

@app.route('/api/artist-detail/<artist_id>')
def get_artist_detail(artist_id):
    """Get artist detail data"""
    try:
        print(f"🎵 Getting artist detail for ID: {artist_id}")

        # Get database instance
        database = get_database()

        # Get artist discography from database
        db_result = database.get_artist_discography(artist_id)

        if not db_result.get('success'):
            print(f"❌ Database returned error: {db_result}")
            return jsonify({
                "success": False,
                "error": db_result.get('error', 'Artist not found')
            }), 404

        artist_info = db_result['artist']
        owned_releases = db_result['owned_releases']

        print(f"✅ Found artist: {artist_info['name']} with {len(owned_releases['albums'])} albums")

        # Fix artist image URL
        print(f"🖼️ Artist image before fix: '{artist_info.get('image_url')}'")
        if artist_info.get('image_url'):
            artist_info['image_url'] = fix_artist_image_url(artist_info['image_url'])
            print(f"🖼️ Artist image after fix: '{artist_info['image_url']}'")
        else:
            print(f"🖼️ No artist image URL found for {artist_info['name']}")

        # Debug final artist data being sent
        print(f"🖼️ Final artist data being sent: {artist_info}")

        # Fix image URLs for all albums
        for album in owned_releases['albums']:
            if album.get('image_url'):
                album['image_url'] = fix_artist_image_url(album['image_url'])

        # Fix image URLs for EPs and singles (currently empty but for future use)
        for ep in owned_releases['eps']:
            if ep.get('image_url'):
                ep['image_url'] = fix_artist_image_url(ep['image_url'])

        for single in owned_releases['singles']:
            if single.get('image_url'):
                single['image_url'] = fix_artist_image_url(single['image_url'])

        # Get Spotify discography for proper categorization and missing releases
        spotify_artist_data = None
        try:
            spotify_discography = get_spotify_artist_discography(artist_info['name'])

            if spotify_discography['success']:
                print(f"🎵 Spotify discography found - Albums: {len(spotify_discography['albums'])}, EPs: {len(spotify_discography['eps'])}, Singles: {len(spotify_discography['singles'])}")

                # Store Spotify artist data for the response
                spotify_artist_data = {
                    'spotify_artist_id': spotify_discography.get('spotify_artist_id'),
                    'spotify_artist_name': spotify_discography.get('spotify_artist_name'),
                    'artist_image': spotify_discography.get('artist_image')
                }

                # Merge owned and Spotify data for complete picture
                merged_discography = merge_discography_data(owned_releases, spotify_discography, db=database, artist_name=artist_info['name'])
            else:
                print(f"⚠️ Spotify discography not found: {spotify_discography.get('error', 'Unknown error')}")
                # Fall back to our database categorization
                merged_discography = owned_releases
        except Exception as spotify_error:
            print(f"⚠️ Error fetching Spotify data: {spotify_error}")
            # Fall back to our database categorization
            merged_discography = owned_releases

        response_data = {
            "success": True,
            "artist": artist_info,
            "discography": merged_discography
        }

        # Add Spotify artist data if available
        if spotify_artist_data:
            response_data["spotify_artist"] = spotify_artist_data

        return jsonify(response_data)

    except Exception as e:
        print(f"❌ Error in get_artist_detail: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({
            "success": False,
            "error": str(e)
        }), 500

@app.route('/api/library/debug-photos')
def debug_library_photos():
    """Debug endpoint to check artist photo URLs"""
    try:
        database = get_database()

        with database._get_connection() as conn:
            cursor = conn.cursor()

            # Get first 10 artists with their photo URLs
            cursor.execute("""
                SELECT name, thumb_url, server_source
                FROM artists
                WHERE thumb_url IS NOT NULL AND thumb_url != ''
                LIMIT 10
            """)

            artists_with_photos = cursor.fetchall()

            # Get first 10 artists without photos
            cursor.execute("""
                SELECT name, thumb_url, server_source
                FROM artists
                WHERE thumb_url IS NULL OR thumb_url = ''
                LIMIT 10
            """)

            artists_without_photos = cursor.fetchall()

            return jsonify({
                "artists_with_photos": [dict(row) for row in artists_with_photos],
                "artists_without_photos": [dict(row) for row in artists_without_photos],
                "total_with_photos": len(artists_with_photos),
                "total_without_photos": len(artists_without_photos)
            })

    except Exception as e:
        return jsonify({"error": str(e)}), 500

@app.route('/api/artist/similar/<path:artist_name>/stream')
def get_similar_artists_stream(artist_name):
    """
    Stream similar artists from MusicMap and match them to Spotify one by one

    Args:
        artist_name: The artist name to find similar artists for

    Returns:
        Server-Sent Events stream with each matched artist
    """
    def generate():
        try:
            print(f"🎵 Streaming similar artists for: {artist_name}")

            # Import required libraries
            from bs4 import BeautifulSoup

            # Construct MusicMap URL
            url_artist = artist_name.lower().replace(' ', '+')
            musicmap_url = f'https://www.music-map.com/{url_artist}'

            print(f"🌐 Fetching MusicMap: {musicmap_url}")

            # Set headers to mimic a browser
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
                'Accept-Language': 'en-US,en;q=0.5',
            }

            # Fetch MusicMap page
            response = requests.get(musicmap_url, headers=headers, timeout=10)
            response.raise_for_status()

            # Parse HTML
            soup = BeautifulSoup(response.text, 'html.parser')
            gnod_map = soup.find(id='gnodMap')

            if not gnod_map:
                yield f"data: {json.dumps({'error': 'Could not find artist map on MusicMap'})}\n\n"
                return

            # Extract similar artist names
            all_anchors = gnod_map.find_all('a')
            searched_artist_lower = artist_name.lower().strip()

            similar_artist_names = []
            for anchor in all_anchors:
                artist_text = anchor.get_text(strip=True)

                # Skip if this is the searched artist
                if artist_text.lower() == searched_artist_lower:
                    continue

                similar_artist_names.append(artist_text)

            print(f"📦 Found {len(similar_artist_names)} similar artists from MusicMap")

            # Determine metadata source
            use_hydrabase = _is_hydrabase_active()

            if not use_hydrabase:
                if not spotify_client or not spotify_client.is_authenticated():
                    yield f"data: {json.dumps({'error': 'Spotify not authenticated'})}\n\n"
                    return

            # Get the searched artist's ID to exclude them
            searched_artist_id = None
            try:
                if use_hydrabase:
                    searched_results = hydrabase_client.search_artists(artist_name, limit=1)
                else:
                    searched_results = spotify_client.search_artists(artist_name, limit=1)
                if searched_results and len(searched_results) > 0:
                    searched_artist_id = searched_results[0].id
                    print(f"🎯 Searched artist ID: {searched_artist_id}")
            except Exception as e:
                print(f"⚠️ Could not get searched artist ID: {e}")

            # Match each artist one by one and stream results
            max_artists = 20
            matched_count = 0
            seen_artist_ids = set()  # Track seen artist IDs to prevent duplicates

            for artist_name_to_match in similar_artist_names[:max_artists]:
                try:
                    print(f"🔍 Matching: {artist_name_to_match}")

                    # Search for the artist via active metadata source
                    if use_hydrabase:
                        results = hydrabase_client.search_artists(artist_name_to_match, limit=1)
                    else:
                        results = spotify_client.search_artists(artist_name_to_match, limit=1)

                    if results and len(results) > 0:
                        spotify_artist = results[0]

                        # Skip if this is the searched artist
                        if spotify_artist.id == searched_artist_id:
                            print(f"⏭️ Skipping searched artist: {spotify_artist.name}")
                            continue

                        # Skip if we've already seen this artist ID (deduplication)
                        if spotify_artist.id in seen_artist_ids:
                            print(f"⏭️ Skipping duplicate artist: {spotify_artist.name}")
                            continue

                        seen_artist_ids.add(spotify_artist.id)

                        artist_data = {
                            'id': spotify_artist.id,
                            'name': spotify_artist.name,
                            'image_url': spotify_artist.image_url if hasattr(spotify_artist, 'image_url') else None,
                            'genres': spotify_artist.genres if hasattr(spotify_artist, 'genres') else [],
                            'popularity': spotify_artist.popularity if hasattr(spotify_artist, 'popularity') else 0
                        }

                        # Stream this matched artist immediately
                        yield f"data: {json.dumps({'artist': artist_data})}\n\n"
                        matched_count += 1

                        print(f"✅ Matched and streamed: {spotify_artist.name}")
                    else:
                        print(f"❌ No Spotify match found for: {artist_name_to_match}")

                except Exception as match_error:
                    print(f"⚠️ Error matching {artist_name_to_match}: {match_error}")
                    continue

            # Send completion message
            yield f"data: {json.dumps({'complete': True, 'total': matched_count})}\n\n"
            print(f"✅ Streaming complete: {matched_count} artists matched")

        except requests.exceptions.RequestException as e:
            print(f"❌ Error fetching MusicMap: {e}")
            yield f"data: {json.dumps({'error': f'Failed to fetch from MusicMap: {str(e)}'})}\n\n"

        except Exception as e:
            print(f"❌ Error streaming similar artists: {e}")
            import traceback
            traceback.print_exc()
            yield f"data: {json.dumps({'error': str(e)})}\n\n"

    return Response(generate(), mimetype='text/event-stream')

@app.route('/api/artist/similar/<path:artist_name>')
def get_similar_artists(artist_name):
    """
    Get similar artists from MusicMap and match them to Spotify (legacy batch endpoint)

    Args:
        artist_name: The artist name to find similar artists for

    Returns:
        JSON with similar artists matched to Spotify data
    """
    try:
        print(f"🎵 Getting similar artists for: {artist_name}")

        # Import required libraries
        from bs4 import BeautifulSoup

        # Construct MusicMap URL
        url_artist = artist_name.lower().replace(' ', '+')
        musicmap_url = f'https://www.music-map.com/{url_artist}'

        print(f"🌐 Fetching MusicMap: {musicmap_url}")

        # Set headers to mimic a browser
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.5',
        }

        # Fetch MusicMap page
        response = requests.get(musicmap_url, headers=headers, timeout=10)
        response.raise_for_status()

        # Parse HTML
        soup = BeautifulSoup(response.text, 'html.parser')
        gnod_map = soup.find(id='gnodMap')

        if not gnod_map:
            return jsonify({
                "success": False,
                "error": "Could not find artist map on MusicMap"
            }), 404

        # Extract similar artist names
        all_anchors = gnod_map.find_all('a')
        searched_artist_lower = artist_name.lower().strip()

        similar_artist_names = []
        for anchor in all_anchors:
            artist_text = anchor.get_text(strip=True)

            # Skip if this is the searched artist
            if artist_text.lower() == searched_artist_lower:
                continue

            similar_artist_names.append(artist_text)

        print(f"📦 Found {len(similar_artist_names)} similar artists from MusicMap")

        # Determine metadata source
        use_hydrabase = _is_hydrabase_active()

        if not use_hydrabase:
            if not spotify_client or not spotify_client.is_authenticated():
                return jsonify({
                    "success": False,
                    "error": "Spotify not authenticated"
                }), 401

        # Get the searched artist's ID to exclude them
        searched_artist_id = None
        try:
            if use_hydrabase:
                searched_results = hydrabase_client.search_artists(artist_name, limit=1)
            else:
                searched_results = spotify_client.search_artists(artist_name, limit=1)
            if searched_results and len(searched_results) > 0:
                searched_artist_id = searched_results[0].id
                print(f"🎯 Searched artist ID: {searched_artist_id}")
        except Exception as e:
            print(f"⚠️ Could not get searched artist ID: {e}")

        # Match each artist (limit to first 20 for performance)
        matched_artists = []
        max_artists = 20
        seen_artist_ids = set()  # Track seen artist IDs to prevent duplicates

        for artist_name_to_match in similar_artist_names[:max_artists]:
            try:
                print(f"🔍 Matching: {artist_name_to_match}")

                # Search for the artist via active metadata source
                if use_hydrabase:
                    results = hydrabase_client.search_artists(artist_name_to_match, limit=1)
                else:
                    results = spotify_client.search_artists(artist_name_to_match, limit=1)

                if results and len(results) > 0:
                    spotify_artist = results[0]

                    # Skip if this is the searched artist
                    if spotify_artist.id == searched_artist_id:
                        print(f"⏭️ Skipping searched artist: {spotify_artist.name}")
                        continue

                    # Skip if we've already seen this artist ID (deduplication)
                    if spotify_artist.id in seen_artist_ids:
                        print(f"⏭️ Skipping duplicate artist: {spotify_artist.name}")
                        continue

                    seen_artist_ids.add(spotify_artist.id)

                    matched_artists.append({
                        'id': spotify_artist.id,
                        'name': spotify_artist.name,
                        'image_url': spotify_artist.image_url if hasattr(spotify_artist, 'image_url') else None,
                        'genres': spotify_artist.genres if hasattr(spotify_artist, 'genres') else [],
                        'popularity': spotify_artist.popularity if hasattr(spotify_artist, 'popularity') else 0
                    })

                    print(f"✅ Matched: {spotify_artist.name}")
                else:
                    print(f"❌ No Spotify match found for: {artist_name_to_match}")

            except Exception as match_error:
                print(f"⚠️ Error matching {artist_name_to_match}: {match_error}")
                continue

        print(f"✅ Successfully matched {len(matched_artists)} artists to Spotify")

        return jsonify({
            "success": True,
            "artist": artist_name,
            "similar_artists": matched_artists,
            "total_found": len(similar_artist_names),
            "total_matched": len(matched_artists)
        })

    except requests.exceptions.RequestException as e:
        print(f"❌ Error fetching MusicMap: {e}")
        return jsonify({
            "success": False,
            "error": f"Failed to fetch from MusicMap: {str(e)}"
        }), 500

    except Exception as e:
        print(f"❌ Error getting similar artists: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({
            "success": False,
            "error": str(e)
        }), 500

@app.route('/api/artist/<artist_id>/image', methods=['GET'])
def get_artist_image(artist_id):
    """Get artist image URL - used for lazy loading in search results.

    For iTunes, this fetches the artist's first album artwork as a fallback.
    For Spotify, returns the artist's image directly.
    """
    try:
        # Soul IDs from Hydrabase can't be looked up on Spotify/iTunes
        if artist_id.startswith('soul_'):
            return jsonify({"success": True, "image_url": None})

        if spotify_client and spotify_client.is_spotify_authenticated():
            # Use Spotify directly
            artist_data = spotify_client.sp.artist(artist_id)
            if artist_data and artist_data.get('images'):
                image_url = artist_data['images'][0]['url'] if artist_data['images'] else None
                return jsonify({"success": True, "image_url": image_url})
            return jsonify({"success": True, "image_url": None})
        else:
            # Use iTunes fallback - fetch album art
            from core.itunes_client import iTunesClient
            itunes = iTunesClient()
            image_url = itunes._get_artist_image_from_albums(artist_id)
            return jsonify({"success": True, "image_url": image_url})
    except Exception as e:
        print(f"Error fetching artist image: {e}")
        return jsonify({"success": False, "image_url": None, "error": str(e)})

@app.route('/api/artist/<artist_id>/discography', methods=['GET'])
def get_artist_discography(artist_id):
    """Get an artist's complete discography (albums and singles)"""
    try:
        # Get optional artist name for fallback searches
        artist_name = request.args.get('artist_name', '')

        # Mirror to Hydrabase P2P network
        if hydrabase_worker and dev_mode_enabled and artist_name:
            hydrabase_worker.enqueue(artist_name, 'discography')

        # Determine which source to use
        spotify_available = spotify_client and spotify_client.is_spotify_authenticated()

        # Import iTunes client for fallback
        from core.itunes_client import iTunesClient
        itunes_client = iTunesClient()

        print(f"🎤 Fetching discography for artist: {artist_id} (name: {artist_name}, spotify: {spotify_available})")

        albums = []
        active_source = None

        # Try Hydrabase first when active
        if _is_hydrabase_active() and artist_name:
            try:
                albums = hydrabase_client.search_discography(artist_name, limit=50)
                if albums:
                    active_source = 'hydrabase'
                    print(f"📊 Got {len(albums)} albums from Hydrabase")
            except Exception as e:
                print(f"Hydrabase discography failed: {e}")

        # Try to get albums from the appropriate source
        # Check if the ID looks like Spotify (alphanumeric) or iTunes (numeric only)
        is_numeric_id = artist_id.isdigit()

        if not albums and spotify_available and not is_numeric_id:
            # Try Spotify first for alphanumeric IDs
            try:
                albums = spotify_client.get_artist_albums(artist_id, album_type='album,single')
                if albums:
                    active_source = 'spotify'
                    print(f"📊 Got {len(albums)} albums from Spotify")
            except Exception as e:
                print(f"Spotify lookup failed: {e}")

        # Try iTunes if Spotify didn't work or if it's a numeric ID
        if not albums:
            try:
                if is_numeric_id:
                    # It's an iTunes ID, use directly
                    albums = itunes_client.get_artist_albums(artist_id, album_type='album,single', limit=50)
                    if albums:
                        active_source = 'itunes'
                        print(f"📊 Got {len(albums)} albums from iTunes (direct ID)")
                elif artist_name:
                    # Search iTunes by name
                    print(f"🔄 Trying iTunes search by name: '{artist_name}'")
                    itunes_artists = itunes_client.search_artists(artist_name, limit=5)
                    if itunes_artists:
                        # Find best match
                        best_match = None
                        for artist in itunes_artists:
                            if artist.name.lower() == artist_name.lower():
                                best_match = artist
                                break
                        if not best_match:
                            best_match = itunes_artists[0]

                        print(f"✅ Found iTunes artist: {best_match.name} (ID: {best_match.id})")
                        albums = itunes_client.get_artist_albums(best_match.id, album_type='album,single', limit=50)
                        if albums:
                            active_source = 'itunes'
                            print(f"📊 Got {len(albums)} albums from iTunes (name search)")
            except Exception as e:
                print(f"iTunes lookup failed: {e}")

        print(f"📊 Total albums returned: {len(albums)} (source: {active_source})")
        
        if not albums:
            return jsonify({
                "albums": [],
                "singles": [],
                "source": active_source or "unknown"
            })
        
        # Separate albums from singles/EPs
        album_list = []
        singles_list = []
        
        # Track seen albums to avoid duplicates (especially for "appears_on")
        seen_albums = set()
        
        for album in albums:
            # Skip duplicates
            if album.id in seen_albums:
                continue
            seen_albums.add(album.id)

            # Skip albums where this artist isn't the primary (first-listed) artist
            if hasattr(album, 'artist_ids') and album.artist_ids:
                if album.artist_ids[0] != artist_id:
                    continue
            
            album_data = {
                "id": album.id,
                "name": album.name,
                "release_date": album.release_date if hasattr(album, 'release_date') else None,
                "album_type": album.album_type if hasattr(album, 'album_type') else 'album',
                "image_url": album.image_url if hasattr(album, 'image_url') else None,
                "total_tracks": album.total_tracks if hasattr(album, 'total_tracks') else 0,
                "external_urls": album.external_urls if hasattr(album, 'external_urls') else {}
            }
            
            # Skip obvious compilation issues but be more lenient for now
            if hasattr(album, 'album_type') and album.album_type == 'compilation':
                print(f"📀 Found compilation: '{album.name}' - including for now")
            
            # Categorize by album type
            if hasattr(album, 'album_type'):
                if album.album_type in ['single', 'ep']:
                    singles_list.append(album_data)
                else:  # 'album' or approved 'compilation'
                    album_list.append(album_data)
            else:
                # Default to album if no type specified
                album_list.append(album_data)
        
        # Sort by release date (newest first)
        def get_release_year(item):
            if item['release_date']:
                try:
                    # Handle different date formats (YYYY, YYYY-MM, YYYY-MM-DD)
                    return int(item['release_date'][:4])
                except (ValueError, IndexError):
                    return 0
            return 0
        
        album_list.sort(key=get_release_year, reverse=True)
        singles_list.sort(key=get_release_year, reverse=True)
        
        print(f"✅ Found {len(album_list)} albums and {len(singles_list)} singles for artist {artist_id}")
        
        # Debug: Log the final album list
        for album in album_list:
            print(f"📀 Album: {album['name']} ({album['album_type']}) - {album['release_date']}")
        for single in singles_list:
            print(f"🎵 Single/EP: {single['name']} ({single['album_type']}) - {single['release_date']}")
        
        return jsonify({
            "albums": album_list,
            "singles": singles_list,
            "source": active_source or "spotify"
        })

    except Exception as e:
        print(f"❌ Error fetching artist discography: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"error": str(e)}), 500

def _resolve_db_album_id(album_id, artist_id=None):
    """Resolve a database album ID to a real Spotify/iTunes album ID.

    When the artist detail page falls back to owned_releases (Spotify artist
    search failed), the album cards carry a database auto-increment ID.
    This helper looks up stored external IDs first, then falls back to an
    iTunes/Spotify search by album title + artist name.
    """
    try:
        database = get_database()
        with database._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                SELECT a.title, a.spotify_album_id, a.itunes_album_id, ar.name as artist_name
                FROM albums a
                JOIN artists ar ON a.artist_id = ar.id
                WHERE a.id = ?
            """, (album_id,))
            row = cursor.fetchone()
            if not row:
                return None

            # Prefer stored external IDs
            if row['spotify_album_id']:
                return row['spotify_album_id']
            if row['itunes_album_id']:
                return row['itunes_album_id']

            # No stored external ID — search by name
            album_title = row['title']
            artist_name = row['artist_name']
            query = f"{artist_name} {album_title}"
            print(f"🔍 Searching for album by name: '{query}'")
            results = spotify_client.search_albums(query, limit=5)
            if results:
                # Pick the best match (search already ranks by relevance)
                for album in results:
                    if album.name.lower().strip() == album_title.lower().strip():
                        print(f"✅ Found exact album match: {album.name} (ID: {album.id})")
                        return album.id
                # Fall back to first result if no exact title match
                print(f"⚠️ No exact match, using best result: {results[0].name} (ID: {results[0].id})")
                return results[0].id

    except Exception as e:
        print(f"⚠️ Error resolving DB album ID {album_id}: {e}")
    return None


@app.route('/api/artist/<artist_id>/album/<album_id>/tracks', methods=['GET'])
def get_artist_album_tracks(artist_id, album_id):
    """Get tracks for specific album formatted for download missing tracks modal"""
    try:
        # Try Hydrabase first when active and album name provided
        if _is_hydrabase_active():
            album_name = request.args.get('name', '')
            album_artist = request.args.get('artist', '')
            try:
                hydra_tracks = hydrabase_client.get_album_tracks(album_id, limit=50)
                if hydra_tracks:
                    album_info = {
                        'id': album_id,
                        'name': album_name or hydra_tracks[0].album or '',
                        'image_url': None,
                        'images': [],
                        'release_date': '',
                        'album_type': 'album',
                        'total_tracks': len(hydra_tracks)
                    }
                    formatted_tracks = []
                    for t in hydra_tracks:
                        artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else []
                        formatted_tracks.append({
                            'id': t.id,
                            'name': t.name,
                            'artists': [a if isinstance(a, str) else a for a in artist_list],
                            'duration_ms': t.duration_ms,
                            'track_number': t.track_number or 0,
                            'disc_number': t.disc_number or 1,
                            'explicit': False,
                            'preview_url': t.preview_url,
                            'external_urls': t.external_urls or {},
                            'uri': '',
                            'album': album_info
                        })
                    print(f"✅ Hydrabase returned {len(formatted_tracks)} tracks for album: {album_info['name']}")
                    return jsonify({
                        'success': True,
                        'album': album_info,
                        'tracks': formatted_tracks
                    })
            except Exception as e:
                logger.warning(f"Hydrabase album_tracks failed for '{album_id}', falling back to Spotify: {e}")

        if not spotify_client or not spotify_client.is_authenticated():
            return jsonify({"error": "Spotify not authenticated"}), 401

        print(f"🎵 Fetching tracks for album: {album_id} by artist: {artist_id}")

        # Get album information first
        album_data = spotify_client.get_album(album_id)
        resolved_album_id = album_id

        # If direct lookup failed, the album_id might be a database ID — resolve it
        if not album_data:
            resolved_album_id = _resolve_db_album_id(album_id, artist_id)
            if resolved_album_id and resolved_album_id != album_id:
                print(f"🔄 Resolved DB album ID {album_id} -> external ID {resolved_album_id}")
                album_data = spotify_client.get_album(resolved_album_id)

        if not album_data:
            return jsonify({"error": "Album not found"}), 404

        # Get album tracks
        tracks_data = spotify_client.get_album_tracks(resolved_album_id)
        if not tracks_data or 'items' not in tracks_data:
            return jsonify({"error": "No tracks found for album"}), 404
        
        # Handle both dict and object responses from spotify_client.get_album()
        if isinstance(album_data, dict):
            album_info = {
                'id': album_data.get('id'),
                'name': album_data.get('name'),
                'image_url': album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None,
                'images': album_data.get('images', []),  # Include images array for wishlist cover art
                'release_date': album_data.get('release_date'),
                'album_type': album_data.get('album_type'),
                'total_tracks': album_data.get('total_tracks')
            }
        else:
            # Handle Album object case
            album_info = {
                'id': album_data.id,
                'name': album_data.name,
                'image_url': album_data.image_url,
                'images': album_data.images if hasattr(album_data, 'images') else [],  # Include images array for wishlist cover art
                'release_date': album_data.release_date,
                'album_type': album_data.album_type,
                'total_tracks': album_data.total_tracks
            }
        
        # Format tracks for download missing tracks modal compatibility
        formatted_tracks = []
        for track_item in tracks_data['items']:
            # Create track object compatible with download missing tracks modal
            formatted_track = {
                'id': track_item['id'],
                'name': track_item['name'],
                'artists': [artist['name'] for artist in track_item['artists']],
                'duration_ms': track_item['duration_ms'],
                'track_number': track_item['track_number'],
                'disc_number': track_item.get('disc_number', 1),
                'explicit': track_item.get('explicit', False),
                'preview_url': track_item.get('preview_url'),
                'external_urls': track_item.get('external_urls', {}),
                'uri': track_item['uri'],
                # Add album context for virtual playlist
                'album': album_info
            }
            formatted_tracks.append(formatted_track)
        
        print(f"✅ Successfully formatted {len(formatted_tracks)} tracks for album: {album_info['name']}")
        
        return jsonify({
            'success': True,
            'album': album_info,
            'tracks': formatted_tracks
        })
        
    except Exception as e:
        print(f"❌ Error fetching album tracks: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"error": str(e)}), 500

@app.route('/api/artist/<artist_id>/completion', methods=['POST'])
def check_artist_discography_completion(artist_id):
    """Check completion status for artist's albums and singles"""
    try:
        data = request.get_json()
        if not data or 'discography' not in data:
            return jsonify({"error": "Missing discography data"}), 400
        
        discography = data['discography']
        test_mode = data.get('test_mode', False)  # Add test mode for demonstration
        albums_completion = []
        singles_completion = []
        
        # Get database instance
        from database.music_database import MusicDatabase
        db = MusicDatabase()
        
        # Get artist name - should be provided by the frontend
        artist_name = data.get('artist_name', 'Unknown Artist')
        
        # If no artist name provided, try to infer it from the request
        if artist_name == 'Unknown Artist':
            print(f"⚠️ No artist name provided in request, attempting to infer from discography data")
            # Try to extract from first album's title by using a simple search
            all_items = discography.get('albums', []) + discography.get('singles', [])
            if all_items and spotify_client and spotify_client.is_authenticated():
                try:
                    first_item = all_items[0]
                    # Search for the first track to get artist name
                    search_results = spotify_client.search_tracks(first_item.get('name', ''), limit=1)
                    if search_results and len(search_results) > 0:
                        artist_name = search_results[0].artists[0] if search_results[0].artists else "Unknown Artist"
                        print(f"🎤 Inferred artist name from search: {artist_name}")
                except Exception as e:
                    print(f"⚠️ Could not infer artist name: {e}")
                    artist_name = "Unknown Artist"
        
        print(f"🎤 Checking completion for artist: {artist_name}")
        
        # Process albums
        for album in discography.get('albums', []):
            completion_data = _check_album_completion(db, album, artist_name, test_mode)
            albums_completion.append(completion_data)
        
        # Process singles/EPs
        for single in discography.get('singles', []):
            completion_data = _check_single_completion(db, single, artist_name, test_mode)
            singles_completion.append(completion_data)
        
        return jsonify({
            "albums": albums_completion,
            "singles": singles_completion
        })
        
    except Exception as e:
        print(f"❌ Error checking discography completion: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"error": str(e)}), 500

def _check_album_completion(db, album_data: dict, artist_name: str, test_mode: bool = False) -> dict:
    """Check completion status for a single album"""
    try:
        album_name = album_data.get('name', '')
        total_tracks = album_data.get('total_tracks', 0)
        album_id = album_data.get('id', '')
        
        print(f"🔍 Checking album: '{album_name}' ({total_tracks} tracks)")
        
        formats = []
        if test_mode:
            # Generate test data to demonstrate the feature
            import random
            owned_tracks = random.randint(0, max(1, total_tracks))
            expected_tracks = total_tracks
            confidence = random.uniform(0.7, 1.0)
            db_album = True  # Simulate found album
            print(f"🧪 TEST MODE: Simulating {owned_tracks}/{expected_tracks} tracks for '{album_name}'")
        else:
            # Check if album exists in database with completeness info
            try:
                # Get active server for database checking
                active_server = config_manager.get_active_media_server()
                db_album, confidence, owned_tracks, expected_tracks, is_complete, formats = db.check_album_exists_with_completeness(
                    title=album_name,
                    artist=artist_name,
                    expected_track_count=total_tracks if total_tracks > 0 else None,
                    confidence_threshold=0.7,  # Slightly lower threshold for better matching
                    server_source=active_server  # Check only the active server
                )
            except Exception as db_error:
                print(f"⚠️ Database error for album '{album_name}': {db_error}")
                # Return error state for this album
                return {
                    "id": album_id,
                    "name": album_name,
                    "status": "error",
                    "owned_tracks": 0,
                    "expected_tracks": total_tracks,
                    "completion_percentage": 0,
                    "confidence": 0.0,
                    "found_in_db": False,
                    "error_message": str(db_error),
                    "formats": []
                }
        
        # Calculate completion percentage
        if expected_tracks > 0:
            completion_percentage = (owned_tracks / expected_tracks) * 100
        elif total_tracks > 0:
            completion_percentage = (owned_tracks / total_tracks) * 100
        else:
            completion_percentage = 100 if owned_tracks > 0 else 0
        
        # Determine completion status based on percentage
        if completion_percentage >= 90 and owned_tracks > 0:
            status = "completed"
        elif completion_percentage >= 60:
            status = "nearly_complete"
        elif completion_percentage > 0:
            status = "partial"
        else:
            status = "missing"
        
        print(f"  📊 Result: {owned_tracks}/{expected_tracks or total_tracks} tracks ({completion_percentage:.1f}%) - {status}")
        
        return {
            "id": album_id,
            "name": album_name,
            "status": status,
            "owned_tracks": owned_tracks,
            "expected_tracks": expected_tracks or total_tracks,
            "completion_percentage": round(completion_percentage, 1),
            "confidence": round(confidence, 2) if confidence else 0.0,
            "found_in_db": db_album is not None,
            "formats": formats
        }

    except Exception as e:
        print(f"❌ Error checking album completion for '{album_data.get('name', 'Unknown')}': {e}")
        return {
            "id": album_data.get('id', ''),
            "name": album_data.get('name', 'Unknown'),
            "status": "error",
            "owned_tracks": 0,
            "expected_tracks": album_data.get('total_tracks', 0),
            "completion_percentage": 0,
            "confidence": 0.0,
            "found_in_db": False,
            "formats": []
        }

def _check_single_completion(db, single_data: dict, artist_name: str, test_mode: bool = False) -> dict:
    """Check completion status for a single/EP (treat EPs like albums, singles as single tracks)"""
    try:
        single_name = single_data.get('name', '')
        total_tracks = single_data.get('total_tracks', 1)
        single_id = single_data.get('id', '')
        album_type = single_data.get('album_type', 'single')
        formats = []

        print(f"🎵 Checking {album_type}: '{single_name}' ({total_tracks} tracks)")

        if test_mode:
            # Generate test data for singles/EPs
            import random
            if album_type == 'ep' or total_tracks > 1:
                owned_tracks = random.randint(0, total_tracks)
                expected_tracks = total_tracks
                confidence = random.uniform(0.7, 1.0)
                print(f"🧪 TEST MODE: EP with {owned_tracks}/{expected_tracks} tracks")
            else:
                owned_tracks = random.choice([0, 1])  # 50/50 chance
                expected_tracks = 1
                confidence = random.uniform(0.7, 1.0) if owned_tracks else 0.0
                print(f"🧪 TEST MODE: Single with {owned_tracks}/{expected_tracks} tracks")
        elif album_type == 'ep' or total_tracks > 1:
            # Treat EPs like albums
            try:
                # Get active server for database checking
                active_server = config_manager.get_active_media_server()
                db_album, confidence, owned_tracks, expected_tracks, is_complete, formats = db.check_album_exists_with_completeness(
                    title=single_name,
                    artist=artist_name,
                    expected_track_count=total_tracks,
                    confidence_threshold=0.7,
                    server_source=active_server  # Check only the active server
                )
            except Exception as db_error:
                print(f"⚠️ Database error for EP '{single_name}': {db_error}")
                owned_tracks, expected_tracks, confidence = 0, total_tracks, 0.0

            # Calculate completion percentage
            if expected_tracks > 0:
                completion_percentage = (owned_tracks / expected_tracks) * 100
            else:
                completion_percentage = (owned_tracks / total_tracks) * 100

            # Determine status
            if completion_percentage >= 90 and owned_tracks > 0:
                status = "completed"
            elif completion_percentage >= 60:
                status = "nearly_complete"
            elif completion_percentage > 0:
                status = "partial"
            else:
                status = "missing"

            print(f"  📊 EP Result: {owned_tracks}/{expected_tracks or total_tracks} tracks ({completion_percentage:.1f}%) - {status}")

        else:
            # Single track - just check if the track exists
            try:
                db_track, confidence = db.check_track_exists(
                    title=single_name,
                    artist=artist_name,
                    confidence_threshold=0.7
                )
            except Exception as db_error:
                print(f"⚠️ Database error for single '{single_name}': {db_error}")
                db_track, confidence = None, 0.0

            owned_tracks = 1 if db_track else 0
            expected_tracks = 1
            completion_percentage = 100 if db_track else 0

            status = "completed" if db_track else "missing"

            # Extract format from single track
            if db_track and db_track.file_path:
                import os
                ext = os.path.splitext(db_track.file_path)[1].lstrip('.').upper()
                if ext == 'MP3' and db_track.bitrate:
                    formats = [f"MP3-{db_track.bitrate}"]
                elif ext:
                    formats = [ext]

            print(f"  🎵 Single Result: {owned_tracks}/1 tracks ({completion_percentage:.1f}%) - {status}")

        return {
            "id": single_id,
            "name": single_name,
            "status": status,
            "owned_tracks": owned_tracks,
            "expected_tracks": expected_tracks or total_tracks,
            "completion_percentage": round(completion_percentage, 1),
            "confidence": round(confidence, 2) if confidence else 0.0,
            "found_in_db": (db_album if album_type == 'ep' or total_tracks > 1 else db_track) is not None,
            "type": album_type,
            "formats": formats
        }

    except Exception as e:
        print(f"❌ Error checking single/EP completion for '{single_data.get('name', 'Unknown')}': {e}")
        return {
            "id": single_data.get('id', ''),
            "name": single_data.get('name', 'Unknown'),
            "status": "error",
            "owned_tracks": 0,
            "expected_tracks": single_data.get('total_tracks', 1),
            "completion_percentage": 0,
            "confidence": 0.0,
            "found_in_db": False,
            "type": single_data.get('album_type', 'single'),
            "formats": []
        }

@app.route('/api/artist/<artist_id>/completion-stream', methods=['POST'])
def check_artist_discography_completion_stream(artist_id):
    """Stream completion status for artist's albums and singles one by one"""
    # Capture request data BEFORE the generator function
    try:
        data = request.get_json()
        if not data or 'discography' not in data:
            return jsonify({"error": "Missing discography data"}), 400
    except Exception as e:
        return jsonify({"error": "Invalid request data"}), 400
    
    # Extract data for the generator
    discography = data['discography']
    test_mode = data.get('test_mode', False)
    artist_name = data.get('artist_name', 'Unknown Artist')
    
    def generate_completion_stream():
        try:
            print(f"🎤 Starting streaming completion check for artist: {artist_name}")
            
            # Get database instance
            from database.music_database import MusicDatabase
            db = MusicDatabase()
            
            # Process albums one by one
            total_items = len(discography.get('albums', [])) + len(discography.get('singles', []))
            processed_count = 0
            
            # Send initial status
            yield f"data: {json.dumps({'type': 'start', 'total_items': total_items, 'artist_name': artist_name})}\n\n"
            
            # Process albums
            for album in discography.get('albums', []):
                try:
                    completion_data = _check_album_completion(db, album, artist_name, test_mode)
                    completion_data['type'] = 'album_completion'
                    completion_data['container_type'] = 'albums'
                    processed_count += 1
                    completion_data['progress'] = round((processed_count / total_items) * 100, 1)
                    
                    yield f"data: {json.dumps(completion_data)}\n\n"
                    
                    # Small delay to make the streaming effect visible
                    time.sleep(0.1)  # 100ms delay between items
                    
                except Exception as e:
                    error_data = {
                        'type': 'error',
                        'container_type': 'albums',
                        'id': album.get('id', ''),
                        'name': album.get('name', 'Unknown'),
                        'error': str(e)
                    }
                    yield f"data: {json.dumps(error_data)}\n\n"
            
            # Process singles/EPs
            for single in discography.get('singles', []):
                try:
                    completion_data = _check_single_completion(db, single, artist_name, test_mode)
                    completion_data['type'] = 'single_completion'
                    completion_data['container_type'] = 'singles'
                    processed_count += 1
                    completion_data['progress'] = round((processed_count / total_items) * 100, 1)
                    
                    yield f"data: {json.dumps(completion_data)}\n\n"
                    
                    # Small delay to make the streaming effect visible
                    time.sleep(0.1)  # 100ms delay between items
                    
                except Exception as e:
                    error_data = {
                        'type': 'error',
                        'container_type': 'singles',
                        'id': single.get('id', ''),
                        'name': single.get('name', 'Unknown'),
                        'error': str(e)
                    }
                    yield f"data: {json.dumps(error_data)}\n\n"
            
            # Send completion signal
            yield f"data: {json.dumps({'type': 'complete', 'processed_count': processed_count})}\n\n"
            
        except Exception as e:
            print(f"❌ Error in streaming completion check: {e}")
            import traceback
            traceback.print_exc()
            yield f"data: {json.dumps({'type': 'error', 'error': str(e)})}\n\n"
    
    return Response(
        generate_completion_stream(),
        content_type='text/event-stream',
        headers={
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'Access-Control-Allow-Origin': '*',
            'Access-Control-Allow-Headers': 'Cache-Control'
        }
    )

@app.route('/api/library/completion-stream', methods=['POST'])
def library_completion_stream():
    """Stream completion status for library artist detail view - checks ownership per release via SSE"""
    try:
        data = request.get_json()
        if not data or 'artist_name' not in data:
            return jsonify({"error": "Missing artist_name"}), 400
    except Exception as e:
        return jsonify({"error": "Invalid request data"}), 400

    artist_name = data['artist_name']

    def generate():
        try:
            from database.music_database import MusicDatabase
            db = MusicDatabase()

            categories = ['albums', 'eps', 'singles']
            all_items = []
            for cat in categories:
                for item in data.get(cat, []):
                    all_items.append((cat, item))

            yield f"data: {json.dumps({'type': 'start', 'total_items': len(all_items)})}\n\n"

            for i, (category, item) in enumerate(all_items):
                try:
                    # Map Library field names to helper field names
                    mapped = {
                        'id': item.get('spotify_id', ''),
                        'name': item['title'],
                        'total_tracks': item.get('track_count', 0),
                        'album_type': item.get('album_type', 'album')
                    }

                    if category == 'singles':
                        result = _check_single_completion(db, mapped, artist_name)
                    else:
                        result = _check_album_completion(db, mapped, artist_name)

                    result['spotify_id'] = item.get('spotify_id', '')
                    result['category'] = category
                    result['type'] = 'completion'
                    yield f"data: {json.dumps(result)}\n\n"
                except Exception as e:
                    yield f"data: {json.dumps({'type': 'completion', 'category': category, 'spotify_id': item.get('spotify_id', ''), 'status': 'error', 'owned_tracks': 0, 'expected_tracks': item.get('track_count', 0), 'completion_percentage': 0, 'confidence': 0.0, 'error': str(e)})}\n\n"

                time.sleep(0.05)  # 50ms between items for visible streaming

            yield f"data: {json.dumps({'type': 'complete', 'processed_count': len(all_items)})}\n\n"

        except Exception as e:
            print(f"❌ Error in library completion stream: {e}")
            import traceback
            traceback.print_exc()
            yield f"data: {json.dumps({'type': 'error', 'error': str(e)})}\n\n"

    return Response(
        generate(),
        content_type='text/event-stream',
        headers={
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'Access-Control-Allow-Origin': '*',
            'Access-Control-Allow-Headers': 'Cache-Control'
        }
    )

@app.route('/api/library/check-tracks', methods=['POST'])
def library_check_tracks():
    """Check which tracks from a list are already owned in the library.
    Uses a single batch DB query + in-memory fuzzy matching for speed."""
    try:
        data = request.get_json()
        if not data or 'artist_name' not in data or 'tracks' not in data:
            return jsonify({"success": False, "error": "Missing artist_name or tracks"}), 400

        artist_name = data['artist_name']
        tracks = data['tracks']

        from database.music_database import MusicDatabase
        db = MusicDatabase()
        active_server = config_manager.get_active_media_server()

        # Single query: get ALL tracks by this artist from the DB
        db_tracks = db.search_tracks(artist=artist_name, limit=500, server_source=active_server)

        if not db_tracks:
            # No tracks by this artist in DB — none owned
            owned_map = {t.get('name', ''): {"owned": False} for t in tracks if t.get('name')}
            return jsonify({"success": True, "owned_tracks": owned_map})

        # Pre-normalize all DB track titles for fast in-memory comparison
        from difflib import SequenceMatcher
        try:
            from unidecode import unidecode
        except ImportError:
            unidecode = lambda x: x

        def _normalize(text):
            if not text:
                return ""
            return unidecode(text).lower().strip()

        def _clean_title(text):
            import re
            cleaned = _normalize(text)
            # Remove parenthetical/bracket content, dashes, feat/ft, remaster tags
            cleaned = re.sub(r'\s*[\[\(].*?[\]\)]', '', cleaned)
            cleaned = re.sub(r'\s*-\s*', ' ', cleaned)
            cleaned = re.sub(r'\s*feat\..*', '', cleaned)
            cleaned = re.sub(r'\s*featuring.*', '', cleaned)
            cleaned = re.sub(r'\s*ft\..*', '', cleaned)
            cleaned = re.sub(r'\s*\d{4}\s*remaster.*', '', cleaned)
            cleaned = re.sub(r'\s*remaster(ed)?.*', '', cleaned)
            cleaned = re.sub(r'\s+', ' ', cleaned).strip()
            return cleaned

        # Pre-compute normalized DB titles once (keep reference to db_track for metadata)
        db_title_entries = [(_normalize(t.title), _clean_title(t.title), t) for t in db_tracks]

        owned_map = {}
        for track in tracks:
            track_name = track.get('name', '')
            if not track_name:
                continue

            search_norm = _normalize(track_name)
            search_clean = _clean_title(track_name)
            matched_db_track = None

            for db_norm, db_clean, db_track in db_title_entries:
                # Check normalized match first (fast path for exact/near-exact)
                if search_norm == db_norm or search_clean == db_clean:
                    matched_db_track = db_track
                    break
                # Fuzzy match: try both normalized and cleaned
                sim = max(
                    SequenceMatcher(None, search_norm, db_norm).ratio(),
                    SequenceMatcher(None, search_clean, db_clean).ratio()
                )
                if sim >= 0.7:
                    matched_db_track = db_track
                    break

            if matched_db_track:
                import os
                file_ext = os.path.splitext(matched_db_track.file_path or '')[1].lstrip('.').upper() or None
                owned_map[track_name] = {
                    "owned": True,
                    "format": file_ext,
                    "bitrate": matched_db_track.bitrate,
                    "album": getattr(matched_db_track, 'album_title', None)
                }
            else:
                owned_map[track_name] = {"owned": False}

        return jsonify({"success": True, "owned_tracks": owned_map})

    except Exception as e:
        print(f"❌ Error checking track ownership: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/stream/start', methods=['POST'])
def stream_start():
    """Start streaming a track in the background"""
    global stream_background_task

    data = request.get_json()
    if not data:
        return jsonify({"success": False, "error": "No track data provided"}), 400
    
    print(f"🎵 Web UI Stream request for: {data.get('filename')}")
    
    try:
        # Stop any existing streaming task
        if stream_background_task and not stream_background_task.done():
            stream_background_task.cancel()
        
        # Reset stream state
        with stream_lock:
            stream_state.update({
                "status": "stopped",
                "progress": 0,
                "track_info": None,
                "file_path": None,
                "error_message": None
            })
        
        # Start new background streaming task
        stream_background_task = stream_executor.submit(_prepare_stream_task, data)
        
        return jsonify({"success": True, "message": "Streaming started"})
        
    except Exception as e:
        print(f"❌ Error starting stream: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/stream/status')
def stream_status():
    """Get current streaming status and progress"""
    try:
        with stream_lock:
            # Return copy of current stream state
            return jsonify({
                "status": stream_state["status"],
                "progress": stream_state["progress"],
                "track_info": stream_state["track_info"],
                "error_message": stream_state["error_message"]
            })
    except Exception as e:
        print(f"❌ Error getting stream status: {e}")
        return jsonify({
            "status": "error",
            "progress": 0,
            "track_info": None,
            "error_message": str(e)
        }), 500

@app.route('/stream/audio')
def stream_audio():
    """Serve the audio file from the Stream folder with range request support"""
    try:
        with stream_lock:
            if stream_state["status"] != "ready" or not stream_state["file_path"]:
                return jsonify({"error": "No audio file ready for streaming"}), 404
            
            file_path = stream_state["file_path"]
        
        if not os.path.exists(file_path):
            return jsonify({"error": "Audio file not found"}), 404
        
        print(f"🎵 Serving audio file: {os.path.basename(file_path)}")
        
        # Determine MIME type based on file extension
        file_ext = os.path.splitext(file_path)[1].lower()
        mime_types = {
            '.mp3': 'audio/mpeg',
            '.flac': 'audio/flac',
            '.ogg': 'audio/ogg',
            '.aac': 'audio/aac',
            '.m4a': 'audio/mp4',
            '.wav': 'audio/wav',
            '.opus': 'audio/ogg',
            '.webm': 'audio/webm',
            '.wma': 'audio/x-ms-wma'
        }
        
        mimetype = mime_types.get(file_ext, 'audio/mpeg')
        
        # Get file size
        file_size = os.path.getsize(file_path)
        
        # Handle range requests (important for HTML5 audio seeking)
        range_header = request.headers.get('Range', None)
        if range_header:
            byte_start = 0
            byte_end = file_size - 1
            
            # Parse range header (format: "bytes=start-end")
            try:
                range_match = re.match(r'bytes=(\d*)-(\d*)', range_header)
                if range_match:
                    start_str, end_str = range_match.groups()
                    if start_str:
                        byte_start = int(start_str)
                    if end_str:
                        byte_end = int(end_str)
                    else:
                        # If no end specified, serve from start to end of file
                        byte_end = file_size - 1
            except (ValueError, AttributeError):
                # Invalid range header, serve full file
                pass
            
            # Ensure valid range
            byte_start = max(0, byte_start)
            byte_end = min(file_size - 1, byte_end)
            content_length = byte_end - byte_start + 1
            
            # Create response with partial content
            def generate():
                with open(file_path, 'rb') as f:
                    f.seek(byte_start)
                    remaining = content_length
                    while remaining:
                        chunk_size = min(8192, remaining)  # 8KB chunks
                        chunk = f.read(chunk_size)
                        if not chunk:
                            break
                        remaining -= len(chunk)
                        yield chunk
            
            response = Response(generate(), 
                              status=206,  # Partial Content
                              mimetype=mimetype,
                              direct_passthrough=True)
            
            # Set range headers
            response.headers.add('Content-Range', f'bytes {byte_start}-{byte_end}/{file_size}')
            response.headers.add('Accept-Ranges', 'bytes')
            response.headers.add('Content-Length', str(content_length))
            response.headers.add('Cache-Control', 'no-cache')
            
            return response
        else:
            # No range request, serve entire file
            response = send_file(file_path, as_attachment=False, mimetype=mimetype)
            response.headers.add('Accept-Ranges', 'bytes')
            response.headers.add('Content-Length', str(file_size))
            response.headers.add('Cache-Control', 'no-cache')
            return response
        
    except Exception as e:
        print(f"❌ Error serving audio file: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/stream/stop', methods=['POST'])
def stream_stop():
    """Stop streaming and clean up"""
    global stream_background_task
    
    try:
        # Cancel background task
        if stream_background_task and not stream_background_task.done():
            stream_background_task.cancel()
        
        # Clear Stream folder
        project_root = os.path.dirname(os.path.abspath(__file__))
        stream_folder = os.path.join(project_root, 'Stream')
        
        if os.path.exists(stream_folder):
            for filename in os.listdir(stream_folder):
                file_path = os.path.join(stream_folder, filename)
                if os.path.isfile(file_path):
                    os.remove(file_path)
                    print(f"🗑️ Removed stream file: {filename}")
        
        # Reset stream state
        with stream_lock:
            stream_state.update({
                "status": "stopped",
                "progress": 0,
                "track_info": None,
                "file_path": None,
                "error_message": None
            })
        
        return jsonify({"success": True, "message": "Stream stopped"})
        
    except Exception as e:
        print(f"❌ Error stopping stream: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

# --- Matched Downloads API Endpoints ---

def _generate_artist_suggestions(search_result, is_album=False, album_result=None):
    """
    Port of ArtistSuggestionThread.generate_artist_suggestions() from GUI
    Generate artist suggestions using multiple strategies
    """
    if not spotify_client or not matching_engine:
        return []
    
    try:
        print(f"🔍 Generating artist suggestions for: {search_result.get('artist', '')} - {search_result.get('title', '')}")
        suggestions = []
        
        # Special handling for albums - use album title to find artist
        if is_album and album_result and album_result.get('album_title'):
            print(f"🎵 Album mode detected - using album title for artist search")
            album_title = album_result.get('album_title', '')
            
            # Clean album title (remove year prefixes like "(2005)")
            import re
            clean_album_title = re.sub(r'^\(\d{4}\)\s*', '', album_title).strip()
            print(f"    clean_album_title: '{clean_album_title}'")
            
            # Search tracks using album title to find the artist
            tracks = spotify_client.search_tracks(clean_album_title, limit=10)
            print(f"📊 Found {len(tracks)} tracks from album search")
            
            # Collect unique artists and their associated tracks/albums
            unique_artists = {}  # artist_name -> list of (track, album) tuples
            for track in tracks:
                for artist_name in track.artists:
                    if artist_name not in unique_artists:
                        unique_artists[artist_name] = []
                    unique_artists[artist_name].append((track, track.album))
            
            # Batch fetch artist objects for speed
            from concurrent.futures import ThreadPoolExecutor, as_completed
            artist_objects = {}  # artist_name -> Artist object
            
            def fetch_artist(artist_name):
                try:
                    matches = spotify_client.search_artists(artist_name, limit=1)
                    if matches:
                        return artist_name, matches[0]
                except Exception as e:
                    print(f"⚠️ Error fetching artist '{artist_name}': {e}")
                return artist_name, None
            
            # Use limited concurrency to respect rate limits
            with ThreadPoolExecutor(max_workers=3) as executor:
                future_to_artist = {executor.submit(fetch_artist, name): name for name in unique_artists.keys()}
                
                for future in as_completed(future_to_artist):
                    artist_name, artist_obj = future.result()
                    if artist_obj:
                        artist_objects[artist_name] = artist_obj
            
            # Calculate confidence scores for each artist
            artist_scores = {}
            for artist_name, track_album_pairs in unique_artists.items():
                if artist_name not in artist_objects:
                    continue
                    
                artist = artist_objects[artist_name]
                best_confidence = 0
                
                # Find the best confidence score across all albums for this artist
                for track, album in track_album_pairs:
                    confidence = matching_engine.similarity_score(
                        matching_engine.normalize_string(clean_album_title),
                        matching_engine.normalize_string(album)
                    )
                    if confidence > best_confidence:
                        best_confidence = confidence
                
                artist_scores[artist_name] = (artist, best_confidence)
            
            # Create suggestions from top matches
            for artist_name, (artist, confidence) in sorted(artist_scores.items(), key=lambda x: x[1][1], reverse=True)[:8]:
                suggestions.append({
                    "artist": {
                        "id": artist.id,
                        "name": artist.name,
                        "image_url": getattr(artist, 'image_url', None),
                        "genres": getattr(artist, 'genres', []),
                        "popularity": getattr(artist, 'popularity', 0)
                    },
                    "confidence": confidence
                })
                
        else:
            # Single track mode - search by artist name
            search_artist = search_result.get('artist', '')
            if not search_artist:
                return []
            
            print(f"🎵 Single track mode - searching for artist: '{search_artist}'")
            
            # Search for artists directly
            artist_matches = spotify_client.search_artists(search_artist, limit=10)
            
            for artist in artist_matches:
                # Calculate confidence based on artist name similarity
                confidence = matching_engine.similarity_score(
                    matching_engine.normalize_string(search_artist),
                    matching_engine.normalize_string(artist.name)
                )
                
                suggestions.append({
                    "artist": {
                        "id": artist.id,
                        "name": artist.name,
                        "image_url": getattr(artist, 'image_url', None),
                        "genres": getattr(artist, 'genres', []),
                        "popularity": getattr(artist, 'popularity', 0)
                    },
                    "confidence": confidence
                })
        
        # Sort by confidence and return top results
        suggestions.sort(key=lambda x: x['confidence'], reverse=True)
        return suggestions[:4]
        
    except Exception as e:
        print(f"❌ Error generating artist suggestions: {e}")
        return []

def _generate_album_suggestions(selected_artist, search_result):
    """
    Port of AlbumSuggestionThread logic from GUI
    Generate album suggestions for a selected artist
    """
    if not spotify_client or not matching_engine:
        return []
    
    try:
        print(f"🔍 Generating album suggestions for artist: {selected_artist['name']}")
        
        # Determine target album name from search result
        target_album_name = search_result.get('album', '') or search_result.get('album_title', '')
        if not target_album_name:
            print("⚠️ No album name found in search result")
            return []
        
        # Clean target album name
        import re
        clean_target = re.sub(r'^\(\d{4}\)\s*', '', target_album_name).strip()
        print(f"    target_album: '{clean_target}'")
        
        # Get artist's albums from Spotify
        artist_albums = spotify_client.get_artist_albums(selected_artist['id'])
        print(f"📊 Found {len(artist_albums)} albums for artist")
        
        album_matches = []
        for album in artist_albums:
            # Calculate confidence based on album name similarity
            confidence = matching_engine.similarity_score(
                matching_engine.normalize_string(clean_target),
                matching_engine.normalize_string(album.name)
            )
            
            album_matches.append({
                "album": {
                    "id": album.id,
                    "name": album.name,
                    "release_date": getattr(album, 'release_date', ''),
                    "album_type": getattr(album, 'album_type', 'album'),
                    "image_url": getattr(album, 'image_url', None),
                    "total_tracks": getattr(album, 'total_tracks', 0)
                },
                "confidence": confidence
            })
        
        # Sort by confidence and return top results
        album_matches.sort(key=lambda x: x['confidence'], reverse=True)
        return album_matches[:4]
        
    except Exception as e:
        print(f"❌ Error generating album suggestions: {e}")
        return []

@app.route('/api/match/suggestions', methods=['POST'])
def get_match_suggestions():
    """Get AI-powered suggestions for artist or album matching"""
    try:
        data = request.get_json()
        search_result = data.get('search_result', {})
        context = data.get('context', 'artist')  # 'artist' or 'album'
        
        if context == 'artist':
            is_album = data.get('is_album', False)
            album_result = data.get('album_result', None) if is_album else None
            suggestions = _generate_artist_suggestions(search_result, is_album, album_result)
        elif context == 'album':
            selected_artist = data.get('selected_artist', {})
            suggestions = _generate_album_suggestions(selected_artist, search_result)
        else:
            return jsonify({"error": "Invalid context. Must be 'artist' or 'album'"}), 400
        
        return jsonify({"suggestions": suggestions})
        
    except Exception as e:
        print(f"❌ Error in match suggestions: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/match/search', methods=['POST'])
def search_match():
    """Manual search for artists or albums"""
    try:
        data = request.get_json()
        query = data.get('query', '').strip()
        context = data.get('context', 'artist')  # 'artist' or 'album'
        
        if not query:
            return jsonify({"results": []})

        use_hydrabase = _is_hydrabase_active()

        # Mirror to Hydrabase P2P network (fire-and-forget when not primary)
        if not use_hydrabase and hydrabase_worker and dev_mode_enabled:
            hydrabase_worker.enqueue(query, context)

        if context == 'artist':
            # Search for artists
            if use_hydrabase:
                artist_matches = hydrabase_client.search_artists(query, limit=8)
            else:
                artist_matches = spotify_client.search_artists(query, limit=8)
            results = []
            
            for artist in artist_matches:
                # Calculate confidence based on search similarity
                confidence = matching_engine.similarity_score(
                    matching_engine.normalize_string(query),
                    matching_engine.normalize_string(artist.name)
                )
                
                results.append({
                    "artist": {
                        "id": artist.id,
                        "name": artist.name,
                        "image_url": getattr(artist, 'image_url', None),
                        "genres": getattr(artist, 'genres', []),
                        "popularity": getattr(artist, 'popularity', 0)
                    },
                    "confidence": confidence
                })
            
            return jsonify({"results": results})
            
        elif context == 'album':
            # Search for albums by specific artist
            artist_id = data.get('artist_id')

            if use_hydrabase:
                # Hydrabase: search albums by query directly
                album_matches = hydrabase_client.search_albums(query, limit=20)
            else:
                if not artist_id:
                    return jsonify({"error": "Artist ID required for album search"}), 400
                # Get artist's albums and filter by query
                album_matches = spotify_client.get_artist_albums(artist_id)

            results = []
            for album in album_matches:
                # Calculate confidence based on query similarity
                confidence = matching_engine.similarity_score(
                    matching_engine.normalize_string(query),
                    matching_engine.normalize_string(album.name)
                )

                # Only include results with reasonable similarity
                if confidence > 0.3:
                    results.append({
                        "album": {
                            "id": album.id,
                            "name": album.name,
                            "release_date": getattr(album, 'release_date', ''),
                            "album_type": getattr(album, 'album_type', 'album'),
                            "image_url": getattr(album, 'image_url', None),
                            "total_tracks": getattr(album, 'total_tracks', 0)
                        },
                        "confidence": confidence
                    })

            # Sort by confidence
            results.sort(key=lambda x: x['confidence'], reverse=True)
            return jsonify({"results": results[:8]})
        
        else:
            return jsonify({"error": "Invalid context. Must be 'artist' or 'album'"}), 400
        
    except Exception as e:
        print(f"❌ Error in match search: {e}")
        return jsonify({"error": str(e)}), 500


def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_artist, spotify_album):
    """
    Download album tracks that have been matched to Spotify with full track metadata.
    This provides the best possible metadata enhancement and organization.
    """
    logger.info(f"🎯 Processing enhanced album download for '{spotify_album['name']}' with {len(enhanced_tracks)} matched tracks")

    # Compute total_discs for multi-disc album subfolder support
    total_discs = max((t['spotify_track'].get('disc_number', 1) for t in enhanced_tracks), default=1)
    spotify_album['total_discs'] = total_discs

    started_count = 0

    # Process matched tracks with full Spotify metadata
    for matched_item in enhanced_tracks:
        try:
            slskd_track = matched_item['slskd_track']
            spotify_track = matched_item['spotify_track']

            username = slskd_track.get('username')
            filename = slskd_track.get('filename')
            size = slskd_track.get('size', 0)

            if not username or not filename:
                logger.warning(f"Skipping track with missing username or filename: {slskd_track}")
                continue

            # Start download
            download_id = run_async(soulseek_client.download(username, filename, size))

            if download_id:
                context_key = _make_context_key(username, filename)
                with matched_context_lock:
                    # Create context with FULL Spotify track metadata (like Download Missing Tracks modal)
                    matched_downloads_context[context_key] = {
                        "spotify_artist": spotify_artist,
                        "spotify_album": spotify_album,
                        "track_info": spotify_track,  # Full Spotify track object!
                        "original_search_result": {
                            'username': username,
                            'filename': filename,
                            'size': size,
                            'title': spotify_track['name'],  # Use Spotify title
                            'artist': spotify_artist['name'],
                            'album': spotify_album['name'],
                            'track_number': spotify_track['track_number'],  # Use Spotify track number
                            'disc_number': spotify_track.get('disc_number', 1),
                            'spotify_clean_title': spotify_track['name']  # For filename generation
                        },
                        "is_album_download": True,
                        "has_full_spotify_metadata": True  # Flag for robust processing
                    }

                logger.info(f"✅ Queued matched track: '{spotify_track['name']}' (track #{spotify_track['track_number']})")
                started_count += 1
            else:
                logger.error(f"Failed to queue track: {filename}")

        except Exception as e:
            logger.error(f"Error processing matched track: {e}")
            continue

    # Process unmatched tracks with basic cleanup
    for slskd_track in unmatched_tracks:
        try:
            username = slskd_track.get('username')
            filename = slskd_track.get('filename')
            size = slskd_track.get('size', 0)

            if not username or not filename:
                continue

            download_id = run_async(soulseek_client.download(username, filename, size))

            if download_id:
                context_key = _make_context_key(username, filename)
                with matched_context_lock:
                    # Basic context for unmatched tracks (simple cleanup)
                    matched_downloads_context[context_key] = {
                        'search_result': {
                            'username': username,
                            'filename': filename,
                            'size': size,
                            'is_simple_download': True  # Falls back to simple transfer
                        },
                        'spotify_artist': None,
                        'spotify_album': None,
                        'track_info': None
                    }

                logger.info(f"⚠️ Queued unmatched track (basic cleanup): {filename}")
                started_count += 1

        except Exception as e:
            logger.error(f"Error processing unmatched track: {e}")
            continue

    return started_count


def _start_album_download_tasks(album_result, spotify_artist, spotify_album):
    """
    This final version now fetches the official Spotify tracklist and uses it to
    match and correct the metadata for each individual track before downloading,
    ensuring perfect tagging and naming.
    """
    print(f"🎵 Processing matched album download for '{spotify_album['name']}' with {len(album_result.get('tracks', []))} tracks.")
    
    tracks_to_download = album_result.get('tracks', [])
    if not tracks_to_download:
        print("⚠️ Album result contained no tracks. Aborting.")
        return 0

    # --- THIS IS THE NEW LOGIC ---
    # Fetch the official tracklist from Spotify ONCE for the entire album.
    official_spotify_tracks = _get_spotify_album_tracks(spotify_album)
    if not official_spotify_tracks:
        print("⚠️ Could not fetch official tracklist from Spotify. Metadata may be inaccurate.")
    # --- END OF NEW LOGIC ---

    # Compute total_discs for multi-disc album subfolder support
    if official_spotify_tracks:
        total_discs = max((t.get('disc_number', 1) for t in official_spotify_tracks), default=1)
    else:
        total_discs = 1
    spotify_album['total_discs'] = total_discs

    started_count = 0
    for track_data in tracks_to_download:
        try:
            username = track_data.get('username') or album_result.get('username')
            filename = track_data.get('filename')
            size = track_data.get('size', 0)

            if not username or not filename:
                continue

            # Pre-parse the filename to get a baseline for metadata
            parsed_meta = _parse_filename_metadata(filename)
            
            # --- THIS IS THE CRITICAL MATCHING STEP ---
            # Match the parsed metadata against the official Spotify tracklist
            corrected_meta = _match_track_to_spotify_title(parsed_meta, official_spotify_tracks)
            # --- END OF CRITICAL STEP ---

            # Create a clean context object using the CORRECTED metadata
            individual_track_context = {
                'username': username,
                'filename': filename,
                'size': size,
                'title': corrected_meta.get('title'),
                'artist': corrected_meta.get('artist') or spotify_artist['name'],
                'album': spotify_album['name'],
                'track_number': corrected_meta.get('track_number'),
                'disc_number': corrected_meta.get('disc_number', 1)
            }

            download_id = run_async(soulseek_client.download(username, filename, size))

            if download_id:
                context_key = _make_context_key(username, filename)
                with matched_context_lock:
                    # Enhanced context storage with Spotify clean titles (GUI parity)
                    enhanced_context = individual_track_context.copy()
                    enhanced_context['spotify_clean_title'] = individual_track_context.get('title', '')

                    matched_downloads_context[context_key] = {
                        "spotify_artist": spotify_artist,
                        "spotify_album": spotify_album,
                        "original_search_result": enhanced_context, # Contains corrected data + clean title
                        "is_album_download": True
                    }
                print(f"  + Queued track: {filename} (Matched to: '{corrected_meta.get('title')}')")
                started_count += 1
            else:
                print(f"  - Failed to queue track: {filename}")

        except Exception as e:
            print(f"❌ Error processing track in album batch: {track_data.get('filename')}. Error: {e}")
            continue
            
    return started_count




@app.route('/api/download/matched', methods=['POST'])
def start_matched_download():
    """
    Starts a matched download. Supports:
    1. Enhanced album downloads with full Spotify track metadata
    2. Regular album downloads (fallback)
    3. Single track downloads
    """
    try:
        data = request.get_json()
        download_payload = data.get('search_result', {})
        spotify_artist = data.get('spotify_artist', {})
        spotify_album = data.get('spotify_album', None)
        enhanced_tracks = data.get('enhanced_tracks', [])  # Album: Matched tracks with Spotify data
        unmatched_tracks = data.get('unmatched_tracks', [])  # Album: Tracks that didn't match
        spotify_track = data.get('spotify_track', None)  # Single: Full Spotify track object
        is_single_track = data.get('is_single_track', False)  # Single: Flag for single track

        if not download_payload or not spotify_artist:
            return jsonify({"success": False, "error": "Missing download payload or artist data"}), 400

        # NEW: Enhanced single track with full Spotify metadata
        if is_single_track and spotify_track:
            logger.info(f"🎯 Starting enhanced single track download: '{spotify_track['name']}' by {spotify_artist['name']}")

            username = download_payload.get('username')
            filename = download_payload.get('filename')
            size = download_payload.get('size', 0)

            if not username or not filename:
                return jsonify({"success": False, "error": "Missing username or filename"}), 400

            download_id = run_async(soulseek_client.download(username, filename, size))

            if download_id:
                context_key = _make_context_key(username, filename)
                with matched_context_lock:
                    # Create context with FULL Spotify track metadata (like Download Missing Tracks modal)
                    matched_downloads_context[context_key] = {
                        "spotify_artist": spotify_artist,
                        "spotify_album": spotify_track.get('album'),  # Single's album from Spotify
                        "track_info": spotify_track,  # Full Spotify track object!
                        "original_search_result": {
                            'username': username,
                            'filename': filename,
                            'size': size,
                            'title': spotify_track['name'],
                            'artist': spotify_artist['name'],
                            'album': spotify_track.get('album', {}).get('name', 'Unknown Album'),
                            'track_number': spotify_track.get('track_number', 1),
                            'spotify_clean_title': spotify_track['name']
                        },
                        "is_album_download": False,  # It's a single
                        "has_full_spotify_metadata": True  # Flag for robust processing
                    }

                logger.info(f"✅ Queued enhanced single track: '{spotify_track['name']}'")
                return jsonify({"success": True, "message": "Enhanced single track download started"})
            else:
                return jsonify({"success": False, "error": "Failed to start download via slskd"}), 500

        # NEW: Enhanced album download with track-to-track matching
        if enhanced_tracks:
            logger.info(f"🎯 Starting enhanced album download: {len(enhanced_tracks)} matched tracks, {len(unmatched_tracks)} unmatched")
            started_count = _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_artist, spotify_album)
            if started_count > 0:
                return jsonify({"success": True, "message": f"Queued {started_count} tracks with full Spotify metadata."})
            else:
                return jsonify({"success": False, "error": "Failed to queue any tracks from the album."}), 500

        # Regular album download (fallback if matching fails)
        is_full_album_download = bool(spotify_album and download_payload.get('result_type') == 'album')

        if is_full_album_download:
            started_count = _start_album_download_tasks(download_payload, spotify_artist, spotify_album)
            if started_count > 0:
                return jsonify({"success": True, "message": f"Queued {started_count} tracks for matched album download."})
            else:
                return jsonify({"success": False, "error": "Failed to queue any tracks from the album."}), 500
        else:
            # This block handles BOTH regular singles AND individual tracks from an album card.
            username = download_payload.get('username')
            filename = download_payload.get('filename')
            size = download_payload.get('size', 0)

            if not username or not filename:
                return jsonify({"success": False, "error": "Missing username or filename"}), 400

            parsed_meta = _parse_filename_metadata(filename)
            download_payload['title'] = parsed_meta.get('title') or download_payload.get('title')
            download_payload['artist'] = parsed_meta.get('artist') or download_payload.get('artist')
            
            download_id = run_async(soulseek_client.download(username, filename, size))

            if download_id:
                context_key = _make_context_key(username, filename)
                with matched_context_lock:
                    # THE FIX: We preserve the spotify_album context if it was provided.
                    # For a regular single, spotify_album will be None.
                    # For an album track, it will contain the album's data.
                    # Enhanced context storage with Spotify clean titles (GUI parity)
                    enhanced_payload = download_payload.copy()
                    enhanced_payload['spotify_clean_title'] = download_payload.get('title', '')

                    matched_downloads_context[context_key] = {
                        "spotify_artist": spotify_artist,
                        "spotify_album": spotify_album, # PRESERVE album context
                        "original_search_result": enhanced_payload,
                        "is_album_download": False # It's a single track download, not a full album job.
                    }
                return jsonify({"success": True, "message": "Matched download started"})
            else:
                return jsonify({"success": False, "error": "Failed to start download via slskd"}), 500

    except Exception as e:
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500






def _parse_filename_metadata(filename: str) -> dict:
    """
    A direct port of the metadata parsing logic from the GUI's soulseek_client.py.
    This is the crucial missing step that cleans filenames BEFORE Spotify matching.
    """
    import re
    import os
    
    metadata = {
        'artist': None,
        'title': None,
        'album': None,
        'track_number': None
    }
    
    # Get just the filename without extension and path
    base_name = os.path.splitext(os.path.basename(filename))[0]
    
    # --- Logic from soulseek_client.py ---
    patterns = [
        # Pattern: 01 - Artist - Title
        r'^(?P<track_number>\d{1,2})\s*[-\.]\s*(?P<artist>.+?)\s*[-–]\s*(?P<title>.+)$',
        # Pattern: Artist - Title
        r'^(?P<artist>.+?)\s*[-–]\s*(?P<title>.+)$',
        # Pattern: 01 - Title
        r'^(?P<track_number>\d{1,2})\s*[-\.]\s*(?P<title>.+)$',
    ]
    
    for pattern in patterns:
        match = re.match(pattern, base_name)
        if match:
            match_dict = match.groupdict()
            metadata['track_number'] = int(match_dict['track_number']) if match_dict.get('track_number') else None
            metadata['artist'] = match_dict.get('artist', '').strip() or None
            metadata['title'] = match_dict.get('title', '').strip() or None
            break # Stop after first successful match
            
    # If title is still missing, use the whole base_name
    if not metadata['title']:
        metadata['title'] = base_name.strip()

    # Fallback for underscore formats like 'Artist_Album_01_Title'
    if not metadata['artist'] and '_' in base_name:
        parts = base_name.split('_')
        if len(parts) >= 3:
            # A common pattern is Artist_Album_TrackNum_Title
            if parts[-2].isdigit():
                metadata['artist'] = parts[0].strip()
                metadata['title'] = parts[-1].strip()
                metadata['track_number'] = int(parts[-2])
                metadata['album'] = parts[1].strip()
    
    # Final cleanup on title if it contains the artist
    if metadata['artist'] and metadata['title'] and metadata['artist'].lower() in metadata['title'].lower():
         metadata['title'] = metadata['title'].replace(metadata['artist'], '').lstrip(' -–_').strip()


    # Try to extract album from the full directory path
    if '/' in filename or '\\' in filename:
        path_parts = filename.replace('\\', '/').split('/')
        if len(path_parts) >= 2:
            # The parent directory is often the album
            potential_album = path_parts[-2]
            # Clean common prefixes like '2024 - '
            cleaned_album = re.sub(r'^\d{4}\s*-\s*', '', potential_album).strip()
            metadata['album'] = cleaned_album

    print(f"🧠 Parsed Filename '{base_name}': Artist='{metadata['artist']}', Title='{metadata['title']}', Album='{metadata['album']}', Track#='{metadata['track_number']}'")
    return metadata


# ===================================================================
# NEW POST-PROCESSING HELPERS (Ported from downloads.py)
# ===================================================================

def _sanitize_filename(filename: str) -> str:
    """Sanitize filename for file system compatibility."""
    import re
    sanitized = re.sub(r'[<>:"/\\|?*]', '_', filename)
    sanitized = re.sub(r'\s+', ' ', sanitized).strip()
    return sanitized[:200]

def _sanitize_context_values(context: dict) -> dict:
    """
    Sanitize all string values in context dict for path safety.

    Prevents characters like '/' in artist names (e.g., 'AC/DC')
    from being interpreted as path separators during template substitution.

    Args:
        context: Dictionary with metadata values

    Returns:
        New dictionary with sanitized string values
    """
    sanitized = {}
    for key, value in context.items():
        if isinstance(value, str):
            sanitized[key] = _sanitize_filename(value)
        else:
            sanitized[key] = value
    return sanitized

def _clean_track_title(track_title: str, artist_name: str) -> str:
    """Clean up track title by removing artist prefix and other noise."""
    import re
    original = track_title.strip()
    cleaned = original
    cleaned = re.sub(r'^\d{1,2}[\.\s\-]+', '', cleaned)
    artist_pattern = re.escape(artist_name) + r'\s*-\s*'
    cleaned = re.sub(f'^{artist_pattern}', '', cleaned, flags=re.IGNORECASE)
    cleaned = re.sub(r'^[A-Za-z0-9\.]+\s*-\s*\d{1,2}\s*-\s*', '', cleaned)
    quality_patterns = [r'\s*[\[\(][0-9]+\s*kbps[\]\)]\s*', r'\s*[\[\(]flac[\]\)]\s*', r'\s*[\[\(]mp3[\]\)]\s*']
    for pattern in quality_patterns:
        cleaned = re.sub(pattern, '', cleaned, flags=re.IGNORECASE)
    cleaned = re.sub(r'^[-\s\.]+', '', cleaned)
    cleaned = re.sub(r'[-\s\.]+$', '', cleaned)
    cleaned = re.sub(r'\s+', ' ', cleaned).strip()
    return cleaned if cleaned else original

def _extract_track_number_from_filename(filename: str, title: str = None) -> int:
    """Extract track number from filename or title, returns 1 if not found."""
    import re
    import os
    text_to_check = f"{title or ''} {os.path.splitext(os.path.basename(filename))[0]}"
    match = re.match(r'^\d{1,2}', text_to_check.strip())
    if match:
        return int(match.group(0))
    return 1

def _search_track_in_album_context(original_search: dict, artist: dict) -> dict:
    """
    Searches for a track within its album context to avoid matching promotional singles.
    This is a direct port from downloads.py for web server use.
    """
    try:
        album_name = original_search.get('album')
        track_title = original_search.get('title')
        if not all([album_name, track_title, artist]):
            return None

        clean_album = _clean_track_title(album_name, artist['name']) # Use track cleaner for album too
        clean_track = _clean_track_title(track_title, artist['name'])

        album_query = f"album:\"{clean_album}\" artist:\"{artist['name']}\""
        albums = spotify_client.search_albums(album_query, limit=1)

        if not albums:
            return None

        spotify_album = albums[0]
        album_tracks_data = spotify_client.get_album_tracks(spotify_album.id)
        if not album_tracks_data or 'items' not in album_tracks_data:
            return None

        for track_data in album_tracks_data['items']:
            similarity = matching_engine.similarity_score(
                matching_engine.normalize_string(clean_track),
                matching_engine.normalize_string(track_data['name'])
            )
            if similarity > 0.7:
                print(f"✅ Found track in album context: '{track_data['name']}'")
                return {
                    'is_album': True,
                    'album_name': spotify_album.name,
                    'track_number': track_data['track_number'],
                    'clean_track_name': track_data['name'],
                    'album_image_url': spotify_album.image_url
                }
        return None
    except Exception as e:
        print(f"❌ Error in _search_track_in_album_context: {e}")
        return None




def _detect_album_info_web(context: dict, artist: dict) -> dict:
    """
    Enhanced album detection with GUI parity - multi-priority logic.
    (Updated to match GUI downloads.py logic exactly)
    """
    try:
        # Log available data for debugging (GUI PARITY)
        original_search = context.get("original_search_result", {})
        print(f"\n🔍 [Album Detection] Starting for track: '{original_search.get('title', 'Unknown')}'")
        print(f"📊 [Data Available]:")
        print(f"   - Clean Spotify title: '{original_search.get('spotify_clean_title', 'None')}'")
        print(f"   - Clean Spotify album: '{original_search.get('spotify_clean_album', 'None')}'")
        print(f"   - Filename album: '{original_search.get('album', 'None')}'")
        print(f"   - Artist: '{artist.get('name', 'Unknown')}'")
        print(f"   - Context has clean data: {context.get('has_clean_spotify_data', False)}")
        print(f"   - Is album download: {context.get('is_album_download', False)}")
        spotify_album_context = context.get("spotify_album")
        is_album_download = context.get("is_album_download", False)
        artist_name = artist['name']
        
        print(f"🔍 Album detection for '{original_search.get('title', 'Unknown')}' by '{artist_name}':")
        print(f"    Has album attr: {bool(original_search.get('album'))}")
        if original_search.get('album'):
            print(f"    Album value: '{original_search.get('album')}'")

        # --- THIS IS THE CRITICAL FIX ---
        # If this is part of a matched album download, we TRUST the context data completely.
        # This is the exact logic from downloads.py.
        if is_album_download and spotify_album_context:
            print("✅ Matched Album context found. Prioritizing pre-matched Spotify data.")
            
            # We exclusively use the track number and title that were matched
            # *before* the download started. We do not try to re-parse the filename.
            track_number = original_search.get('track_number', 1)
            clean_track_name = original_search.get('title', 'Unknown Track')

            print(f"   -> Using pre-matched Track #{track_number} and Title '{clean_track_name}'")

            return {
                'is_album': True,
                'album_name': spotify_album_context['name'],
                'track_number': track_number,
                'clean_track_name': clean_track_name,
                'album_image_url': spotify_album_context.get('image_url')
            }

        # PRIORITY 1: Try album-aware search using clean Spotify album name (GUI PARITY)
        # Prioritize clean Spotify album name over filename-parsed album
        clean_album_name = original_search.get('spotify_clean_album')
        fallback_album_name = original_search.get('album')
        
        album_name_to_use = None
        album_source = None
        
        if clean_album_name and clean_album_name.strip() and clean_album_name != "Unknown Album":
            album_name_to_use = clean_album_name
            album_source = "CLEAN_SPOTIFY"
        elif fallback_album_name and fallback_album_name.strip() and fallback_album_name != "Unknown Album":
            album_name_to_use = fallback_album_name
            album_source = "FILENAME_PARSED"
        
        if album_name_to_use:
            track_title = original_search.get('spotify_clean_title') or original_search.get('title', 'Unknown')
            print(f"🎯 ALBUM-AWARE SEARCH ({album_source}): Looking for '{track_title}' in album '{album_name_to_use}'")
            
            # Temporarily set the album for the search
            original_album = original_search.get('album')
            original_search['album'] = album_name_to_use
            
            try:
                album_result = _search_track_in_album_context_web(context, artist)
                if album_result:
                    print(f"✅ PRIORITY 1 SUCCESS: Found track using {album_source} album name - FORCING album classification")
                    return album_result
                else:
                    print(f"⚠️ PRIORITY 1 FAILED: Track not found using {album_source} album name")
            finally:
                # Restore original album value
                if original_album is not None:
                    original_search['album'] = original_album
                else:
                    original_search.pop('album', None)

        # PRIORITY 2: Fallback to individual track search for clean metadata
        print(f"🔍 Searching Spotify for individual track info (PRIORITY 2)...")
        
        # Clean the track title before searching - remove artist prefix  
        # Prioritize clean Spotify title over filename-parsed title
        track_title_to_use = original_search.get('spotify_clean_title') or original_search.get('title', '')
        clean_title = _clean_track_title_web(track_title_to_use, artist_name)
        print(f"🧹 Cleaned title: '{track_title_to_use}' -> '{clean_title}'")
        
        # Search for the track by artist and cleaned title
        query = f"artist:{artist_name} track:{clean_title}"
        tracks = spotify_client.search_tracks(query, limit=5)
        
        # Find the best matching track
        best_match = None
        best_confidence = 0
        
        if tracks:
            from core.matching_engine import MusicMatchingEngine
            matching_engine = MusicMatchingEngine()
            for track in tracks:
                # Calculate confidence based on artist and title similarity
                artist_confidence = matching_engine.similarity_score(
                    matching_engine.normalize_string(artist_name),
                    matching_engine.normalize_string(track.artists[0] if track.artists else '')
                )
                title_confidence = matching_engine.similarity_score(
                    matching_engine.normalize_string(clean_title),
                    matching_engine.normalize_string(track.name)
                )
                
                combined_confidence = (artist_confidence * 0.6 + title_confidence * 0.4)
                
                if combined_confidence > best_confidence and combined_confidence > 0.75:  # Higher threshold to avoid bad matches
                    best_match = track
                    best_confidence = combined_confidence

        # If we found a good Spotify match, use it for clean metadata
        if best_match and best_confidence > 0.75:
            print(f"✅ Found matching Spotify track: '{best_match.name}' - Album: '{best_match.album}' (confidence: {best_confidence:.2f})")
            
            # Get detailed track information using Spotify's track API
            detailed_track = None
            if hasattr(best_match, 'id') and best_match.id:
                print(f"🔍 Getting detailed track info from Spotify API for track ID: {best_match.id}")
                detailed_track = spotify_client.get_track_details(best_match.id)
            
            # Use detailed track data if available
            if detailed_track:
                print(f"✅ Got detailed track data from Spotify API")
                album_name = _clean_album_title_web(detailed_track['album']['name'], artist_name)
                clean_track_name = detailed_track['name']  # Use Spotify's clean track name
                album_type = detailed_track['album'].get('album_type', 'album')
                total_tracks = detailed_track['album'].get('total_tracks', 1)
                spotify_track_number = detailed_track.get('track_number', 1)
                
                print(f"📀 Spotify album info: '{album_name}' (type: {album_type}, total_tracks: {total_tracks}, track#: {spotify_track_number})")
                print(f"🎵 Clean track name from Spotify: '{clean_track_name}'")
                
                # Enhanced album detection using detailed API data (GUI PARITY)
                is_album = (
                    # Album type is 'album' (not 'single')
                    album_type == 'album' and
                    # Album has multiple tracks
                    total_tracks > 1 and
                    # Album name different from track name
                    matching_engine.normalize_string(album_name) != matching_engine.normalize_string(clean_track_name) and
                    # Album name is not just the artist name
                    matching_engine.normalize_string(album_name) != matching_engine.normalize_string(artist_name)
                )
                
                album_image_url = None
                if detailed_track['album'].get('images'):
                    album_image_url = detailed_track['album']['images'][0].get('url')
                
                print(f"📊 Album classification: {is_album} (type={album_type}, tracks={total_tracks})")
                
                return {
                    'is_album': is_album,
                    'album_name': album_name,
                    'track_number': spotify_track_number,
                    'clean_track_name': clean_track_name,
                    'album_image_url': album_image_url,
                    'confidence': best_confidence,
                    'source': 'spotify_api_detailed'
                }

        # Fallback: Use original data with basic cleaning
        print("⚠️ No good Spotify match found, using original data")
        fallback_title = _clean_track_title_web(original_search.get('title', 'Unknown Track'), artist_name)
        
        return {
            'is_album': False,
            'clean_track_name': fallback_title,
            'album_name': fallback_title,
            'track_number': 1,
            'confidence': 0.0,
            'source': 'fallback_original'
        }
        
    except Exception as e:
        print(f"❌ Error in _detect_album_info_web: {e}")
        clean_title = _clean_track_title_web(context.get("original_search_result", {}).get('title', 'Unknown'), artist.get('name', ''))
        return {'is_album': False, 'clean_track_name': clean_title, 'album_name': clean_title, 'track_number': 1}




def _cleanup_empty_directories(download_path, moved_file_path):
    """Cleans up empty directories after a file move, ignoring hidden files."""
    import os
    try:
        current_dir = os.path.dirname(moved_file_path)
        while current_dir != download_path and current_dir.startswith(download_path):
            is_empty = not any(not f.startswith('.') for f in os.listdir(current_dir))
            if is_empty:
                print(f"Removing empty directory: {current_dir}")
                os.rmdir(current_dir)
                current_dir = os.path.dirname(current_dir)
            else:
                break
    except Exception as e:
        print(f"Warning: An error occurred during directory cleanup: {e}")


def _sweep_empty_download_directories():
    """
    Walk the download directory bottom-up and remove ALL empty directories.
    Called periodically when no downloads or post-processing are active.
    Handles the edge case where per-file cleanup misses folders that become
    empty only after all sibling downloads in a batch have been processed.
    """
    import os
    try:
        download_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
        if not os.path.isdir(download_path):
            return 0

        removed = 0
        # os.walk bottom-up: deepest directories first so parents become empty after children removed
        for dirpath, _dirnames, _filenames in os.walk(download_path, topdown=False):
            # Never remove the root download directory itself
            if os.path.normpath(dirpath) == os.path.normpath(download_path):
                continue
            # Re-read actual contents — os.walk's lists are stale after child removal
            try:
                entries = os.listdir(dirpath)
            except OSError:
                continue
            visible = [e for e in entries if not e.startswith('.')]
            if not visible:
                try:
                    # Remove any leftover hidden files (e.g. .DS_Store) before rmdir
                    for hidden in entries:
                        try:
                            os.remove(os.path.join(dirpath, hidden))
                        except Exception:
                            pass
                    os.rmdir(dirpath)
                    removed += 1
                except OSError:
                    pass  # Directory not actually empty or locked — skip silently

        if removed > 0:
            print(f"🧹 [Folder Cleanup] Removed {removed} empty director{'y' if removed == 1 else 'ies'} from downloads folder")
        return removed
    except Exception as e:
        print(f"⚠️ [Folder Cleanup] Error sweeping empty directories: {e}")
        return 0


# ===================================================================
# ALBUM GROUPING SYSTEM (Ported from GUI downloads.py)
# ===================================================================

def _get_base_album_name(album_name: str) -> str:
    """
    Extract the base album name without edition indicators.
    E.g., 'good kid, m.A.A.d city (Deluxe Edition)' -> 'good kid, m.A.A.d city'
         'Battle Hymns (MMXI Special Edition)' -> 'Battle Hymns'
    """
    import re

    # Remove common edition suffixes
    base_name = album_name

    # Remove edition indicators in parentheses or brackets
    # Allow any prefix before the keyword (e.g. "MMXI Special Edition", "20th Anniversary Edition")
    base_name = re.sub(r'\s*[\[\(][^)\]]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^)\]]*[\]\)]\s*$', '', base_name, flags=re.IGNORECASE)

    # Generic: any parenthesized/bracketed text ending with "edition"
    base_name = re.sub(r'\s*[\[\(][^)\]]*\bedition\b[^)\]]*[\]\)]\s*$', '', base_name, flags=re.IGNORECASE)

    # Remove standalone edition words at the end
    base_name = re.sub(r'\s+(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$', '', base_name, flags=re.IGNORECASE)

    return base_name.strip()

def _detect_deluxe_edition(album_name: str) -> bool:
    """
    Detect if an album name indicates a deluxe/special edition.
    Returns True if it's a deluxe variant, False for standard.
    """
    if not album_name:
        return False
    
    album_lower = album_name.lower()
    
    # Check for deluxe indicators
    deluxe_indicators = [
        'deluxe',
        'deluxe edition',
        'special edition',
        'expanded edition',
        'extended edition',
        'bonus',
        'remastered',
        'anniversary',
        'collectors edition',
        'limited edition',
        'silver edition',
        'gold edition',
        'platinum edition',
    ]
    
    for indicator in deluxe_indicators:
        if indicator in album_lower:
            print(f"🎯 Detected deluxe edition: '{album_name}' contains '{indicator}'")
            return True
    
    return False

def _normalize_base_album_name(base_album: str, artist_name: str) -> str:
    """
    Normalize the base album name to handle case variations and known corrections.
    """
    import re
    
    # Apply known album corrections for consistent naming
    normalized_lower = base_album.lower().strip()
    
    # Handle common album title variations
    known_corrections = {
        # Add specific album name corrections here as needed
        # Example: "good kid maad city": "good kid, m.A.A.d city"
    }
    
    # Check for exact matches in our corrections
    for variant, correction in known_corrections.items():
        if normalized_lower == variant.lower():
            print(f"📀 Album correction applied: '{base_album}' -> '{correction}'")
            return correction
    
    # Handle punctuation variations 
    normalized = base_album
    
    # Normalize common punctuation patterns
    normalized = re.sub(r'\s*&\s*', ' & ', normalized)  # Standardize & spacing
    normalized = re.sub(r'\s+', ' ', normalized)  # Clean multiple spaces
    normalized = normalized.strip()
    
    print(f"📀 Album variant normalization: '{base_album}' -> '{normalized}'")
    return normalized

def _resolve_album_group(spotify_artist: dict, album_info: dict, original_album: str = None) -> str:
    """
    Smart album grouping: Start with standard, upgrade to deluxe if ANY track is deluxe.
    This ensures all tracks from the same album get the same folder name.
    (Adapted from GUI downloads.py)
    """
    try:
        with album_cache_lock:
            artist_name = spotify_artist["name"]
            detected_album = album_info.get('album_name', '')
            
            # Extract base album name (without edition indicators)
            if detected_album:
                base_album = _get_base_album_name(detected_album)
            elif original_album:
                # Clean the original Soulseek album name 
                cleaned_original = _clean_album_title_web(original_album, artist_name)
                base_album = _get_base_album_name(cleaned_original)
            else:
                base_album = _get_base_album_name(detected_album)
            
            # Normalize the base name (handle case variations, etc.)
            base_album = _normalize_base_album_name(base_album, artist_name)
            
            # Create a key for this album group (artist + base album)
            album_key = f"{artist_name}::{base_album}"
            
            # Check if we already have a cached result for this album
            if album_key in album_name_cache:
                cached_name = album_name_cache[album_key]
                print(f"🔍 Using cached album name for '{album_key}': '{cached_name}'")
                return cached_name
            
            print(f"🔍 Album grouping - Key: '{album_key}', Detected: '{detected_album}'")
            
            # Check if this track indicates a deluxe edition
            is_deluxe_track = False
            if detected_album:
                is_deluxe_track = _detect_deluxe_edition(detected_album)
            elif original_album:
                is_deluxe_track = _detect_deluxe_edition(original_album)
            
            # Get current edition level for this album group (default to standard)
            current_edition = album_editions.get(album_key, "standard")
            
            # SMART ALGORITHM: Upgrade to deluxe if this track is deluxe
            if is_deluxe_track and current_edition == "standard":
                print(f"🎯 UPGRADE: Album '{base_album}' upgraded from standard to deluxe!")
                album_editions[album_key] = "deluxe"
                current_edition = "deluxe"
            
            # Build final album name based on edition level
            if current_edition == "deluxe":
                final_album_name = f"{base_album} (Deluxe Edition)"
            else:
                final_album_name = base_album
            
            # Store the resolution in both caches
            album_groups[album_key] = final_album_name
            album_name_cache[album_key] = final_album_name
            album_artists[album_key] = artist_name
            
            print(f"🔗 Album resolution: '{detected_album}' -> '{final_album_name}' (edition: {current_edition})")
            
            return final_album_name
        
    except Exception as e:
        print(f"❌ Error resolving album group: {e}")
        return album_info.get('album_name', 'Unknown Album')

def _clean_album_title_web(album_title: str, artist_name: str) -> str:
    """Clean up album title by removing common prefixes, suffixes, and artist redundancy"""
    import re
    
    # Start with the original title
    original = album_title.strip()
    cleaned = original
    print(f"🧹 Album Title Cleaning: '{original}' (artist: '{artist_name}')")
    
    # Remove "Album - " prefix
    cleaned = re.sub(r'^Album\s*-\s*', '', cleaned, flags=re.IGNORECASE)
    
    # Remove artist name prefix if it appears at the beginning
    # This handles cases like "Kendrick Lamar - good kid, m.A.A.d city"
    artist_pattern = re.escape(artist_name) + r'\s*-\s*'
    cleaned = re.sub(f'^{artist_pattern}', '', cleaned, flags=re.IGNORECASE)
    
    # Remove common Soulseek suffixes in square brackets and parentheses
    # Examples: [Deluxe Edition] [2012] [320 Kbps] [Album+iTunes+Bonus Tracks] [F10]
    #           (Deluxe Edition) (2012) (320 Kbps) etc.
    # Remove year patterns like [2012], (2020), etc.
    cleaned = re.sub(r'\s*[\[\(]\d{4}[\]\)]\s*', ' ', cleaned)
    
    # Remove quality/format indicators
    quality_patterns = [
        r'\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*',
        r'\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*',
        r'\s*[\[\(].*?flac.*?[\]\)]\s*',
        r'\s*[\[\(].*?mp3.*?[\]\)]\s*',
        r'\s*[\[\(].*?itunes.*?[\]\)]\s*',
        r'\s*[\[\(].*?web.*?[\]\)]\s*',
        r'\s*[\[\(].*?cd.*?[\]\)]\s*'
    ]
    
    for pattern in quality_patterns:
        cleaned = re.sub(pattern, ' ', cleaned, flags=re.IGNORECASE)
    
    # Remove common edition indicators (but preserve them for deluxe detection above)
    # This happens AFTER deluxe detection to avoid interfering with that logic
    
    # Clean up spacing
    cleaned = re.sub(r'\s+', ' ', cleaned).strip()
    
    # Remove leading/trailing punctuation
    cleaned = re.sub(r'^[-\s]+|[-\s]+$', '', cleaned)
    
    print(f"🧹 Album Title Result: '{original}' -> '{cleaned}'")
    return cleaned if cleaned else original

def _search_track_in_album_context_web(context: dict, spotify_artist: dict) -> dict:
    """
    Search for a track within its album context to avoid promotional single confusion.
    (Ported from GUI downloads.py)
    """
    try:
        from core.matching_engine import MusicMatchingEngine
        matching_engine = MusicMatchingEngine()
        
        # Get album and track info from context
        original_search = context.get("original_search_result", {})
        album_name = original_search.get("album")
        track_title = original_search.get("title")
        artist_name = spotify_artist["name"]
        
        if not album_name or not track_title:
            print(f"❌ Album-aware search failed: Missing album ({album_name}) or track ({track_title})")
            return None
        
        print(f"🎯 Album-aware search: '{track_title}' in album '{album_name}' by '{artist_name}'")
        
        # Clean the album name for better search results
        clean_album = _clean_album_title_web(album_name, artist_name)
        clean_track = _clean_track_title_web(track_title, artist_name)
        
        # Search for the specific album first
        album_query = f"album:{clean_album} artist:{artist_name}"
        print(f"🔍 Searching albums: {album_query}")
        albums = spotify_client.search_albums(album_query, limit=5)
        
        if not albums:
            print(f"❌ No albums found for query: {album_query}")
            return None
        
        # Check each album to see if our track is in it
        for album in albums:
            print(f"🎵 Checking album: '{album.name}' ({album.total_tracks} tracks)")
            
            # Get tracks from this album
            album_tracks_data = spotify_client.get_album_tracks(album.id)
            if not album_tracks_data or 'items' not in album_tracks_data:
                print(f"❌ Could not get tracks for album: {album.name}")
                continue
            
            # Check if our track is in this album
            for track_data in album_tracks_data['items']:
                track_name = track_data['name']
                track_number = track_data['track_number']
                
                # Calculate similarity between our track and this album track
                similarity = matching_engine.similarity_score(
                    matching_engine.normalize_string(clean_track),
                    matching_engine.normalize_string(track_name)
                )
                
                # Use higher threshold for remix matching to ensure precision (GUI PARITY)
                is_remix = any(word in clean_track.lower() for word in ['remix', 'mix', 'edit', 'version'])
                threshold = 0.9 if is_remix else 0.65  # Lower threshold to favor album matches over singles
                
                if similarity > threshold:
                    print(f"✅ FOUND: '{track_name}' (track #{track_number}) matches '{clean_track}' (similarity: {similarity:.2f})")
                    print(f"🎯 Forcing album classification for track in '{album.name}'")
                    
                    # Return album info - force album classification!
                    return {
                        'is_album': True,  # Always true - we found it in an album!
                        'album_name': album.name,
                        'track_number': track_number,
                        'clean_track_name': clean_track,  # Use the ORIGINAL download title, not the database match
                        'album_image_url': album.image_url,
                        'confidence': similarity,
                        'source': 'album_context_search'
                    }
            
            print(f"❌ Track '{clean_track}' not found in album '{album.name}'")
        
        print(f"❌ Track '{clean_track}' not found in any matching albums")
        return None
        
    except Exception as e:
        print(f"❌ Error in album-aware search: {e}")
        return None

def _clean_track_title_web(track_title: str, artist_name: str) -> str:
    """Clean up track title by removing artist prefix and common patterns"""
    import re
    
    # Start with the original title
    original = track_title.strip()
    cleaned = original
    print(f"🧹 Track Title Cleaning: '{original}' (artist: '{artist_name}')")
    
    # Remove artist name prefix if it appears at the beginning
    # This handles cases like "Kendrick Lamar - HUMBLE."
    artist_pattern = re.escape(artist_name) + r'\s*-\s*'
    cleaned = re.sub(f'^{artist_pattern}', '', cleaned, flags=re.IGNORECASE)
    
    # Remove common prefixes
    cleaned = re.sub(r'^Track\s*\d*\s*-\s*', '', cleaned, flags=re.IGNORECASE)
    cleaned = re.sub(r'^\d+\.\s*', '', cleaned)  # Remove track numbers like "01. "
    
    # Remove quality/format indicators
    quality_patterns = [
        r'\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*',
        r'\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*',
        r'\s*[\[\(].*?flac.*?[\]\)]\s*',
        r'\s*[\[\(].*?mp3.*?[\]\)]\s*',
        r'\s*[\[\(].*?explicit.*?[\]\)]\s*'
    ]
    
    for pattern in quality_patterns:
        cleaned = re.sub(pattern, ' ', cleaned, flags=re.IGNORECASE)
    
    # Clean up spacing
    cleaned = re.sub(r'\s+', ' ', cleaned).strip()
    
    # Remove leading/trailing punctuation
    cleaned = re.sub(r'^[-\s]+|[-\s]+$', '', cleaned)
    
    print(f"🧹 Track Title Result: '{original}' -> '{cleaned}'")
    return cleaned if cleaned else original


# ===================================================================
# YOUTUBE TRACK CLEANING FUNCTIONS (Ported from GUI sync.py)
# ===================================================================

def clean_youtube_track_title(title, artist_name=None):
    """
    Aggressively clean YouTube track titles by removing video noise and extracting clean track names
    
    Examples:
    'No Way Jose (Official Music Video)' → 'No Way Jose'
    'bbno$ - mary poppins (official music video)' → 'mary poppins'
    'Beyond (From "Moana 2") (Official Video) ft. Rachel House' → 'Beyond'
    'Temporary (feat. Skylar Grey) [Official Music Video]' → 'Temporary'
    'ALL MY LOVE (Directors\' Cut)' → 'ALL MY LOVE'
    'Espresso Macchiato | Estonia 🇪🇪 | Official Music Video | #Eurovision2025' → 'Espresso Macchiato'
    """
    import re
    
    if not title:
        return title
    
    original_title = title
    
    # FIRST: Try to extract track name from "Artist - Track" or "Track - Artist" format
    artist_removed = False
    if artist_name and '-' in title:
        # Check if artist is at the start: "Artist - Track" or "Artist & Others - Track"
        # Handle collaborations: "Artist1 & Artist2 - Track" or "Artist, Artist2 - Track"
        artist_pattern = r'^' + re.escape(artist_name.strip()) + r'(?:\s*[&,x]\s*[^-]+)?\s*[-–—]\s*'
        cleaned_title = re.sub(artist_pattern, '', title, flags=re.IGNORECASE).strip()

        if cleaned_title != title:
            print(f"🎯 Removed artist from start: '{title}' -> '{cleaned_title}' (artist: '{artist_name}')")
            title = cleaned_title
            artist_removed = True
        else:
            # Artist not at start, check if format is "Track - Artist" by looking for artist at end
            # Only remove trailing artist if it comes after a dash
            artist_end_pattern = r'\s*[-–—]\s*' + re.escape(artist_name.strip()) + r'(?:\s*[&,x]\s*[^-]+)?\s*$'
            cleaned_title = re.sub(artist_end_pattern, '', title, flags=re.IGNORECASE).strip()

            if cleaned_title != title:
                print(f"🎯 Removed artist from end: '{title}' -> '{cleaned_title}' (artist: '{artist_name}')")
                title = cleaned_title
                artist_removed = True

    # Remove content in brackets/braces BEFORE removing dashes
    title = re.sub(r'【[^】]*】', '', title)  # Japanese brackets
    title = re.sub(r'\s*\([^)]*\)', '', title)   # Parentheses - removes everything after first (
    title = re.sub(r'\s*\(.*$', '', title)      # Remove everything after lone ( (unmatched parentheses)
    title = re.sub(r'\[[^\]]*\]', '', title)  # Square brackets
    title = re.sub(r'\{[^}]*\}', '', title)   # Curly braces
    title = re.sub(r'<[^>]*>', '', title)     # Angle brackets

    # ONLY remove trailing dashes with garbage if artist was already removed
    # This prevents "Artist1, Artist2 - Song" from becoming "Artist1, Artist2"
    if artist_removed:
        # Safe to remove any remaining trailing dash content (likely album/extra info)
        title = re.sub(r'\s*-\s*.*$', '', title)
    
    # Remove everything after pipes (|) - often used for additional context
    title = re.split(r'\s*\|\s*', title)[0].strip()
    
    # Remove common video/platform noise
    noise_patterns = [
        r'\bapple\s+music\b',
        r'\bfull\s+video\b', 
        r'\bmusic\s+video\b',
        r'\bofficial\s+video\b',
        r'\bofficial\s+music\s+video\b',
        r'\bofficial\b',
        r'\bcensored\s+version\b',
        r'\buncensored\s+version\b',
        r'\bexplicit\s+version\b',
        r'\blive\s+version\b',
        r'\bversion\b',
        r'\btopic\b',
        r'\baudio\b',
        r'\blyrics?\b',
        r'\blyric\s+video\b',
        r'\bwith\s+lyrics?\b',
        r'\bvisuali[sz]er\b',
        r'\bmv\b',
        r'\bdirectors?\s+cut\b',
        r'\bremaster(ed)?\b',
        r'\bremix\b'
    ]
    
    for pattern in noise_patterns:
        title = re.sub(pattern, '', title, flags=re.IGNORECASE)
    
    # Only remove artist name if it's standalone (not part of "Artist1 & Artist2")
    # Skip this if the title contains collaboration indicators near the artist name
    if artist_name:
        # Check if artist appears with collaboration indicators (& or ,)
        collab_pattern = rf'\b{re.escape(artist_name)}\s*[&,]\s*\w+|[\w\s]+[&,]\s*{re.escape(artist_name)}\b'
        has_collab = re.search(collab_pattern, title, flags=re.IGNORECASE)

        if not has_collab:
            # Safe to remove artist - it's standalone
            title = re.sub(rf'\b{re.escape(artist_name)}\b', '', title, flags=re.IGNORECASE)
            title = re.sub(rf'\b{re.escape(artist_name)}\s*[-–—:]\s*', '', title, flags=re.IGNORECASE)
            title = re.sub(rf'^{re.escape(artist_name)}\s*[-–—:]\s*', '', title, flags=re.IGNORECASE)
        else:
            print(f"⚠️ Skipping artist removal - collaboration detected: '{title}'")
    
    # Remove "prod. Producer" patterns
    title = re.sub(r'\s+prod\.?\s+\S+', '', title, flags=re.IGNORECASE)

    # Remove all quotes and other punctuation
    title = re.sub(r'["\'''""„‚‛‹›«»]', '', title)

    # Remove featured artist patterns (after removing parentheses)
    feat_patterns = [
        r'\s+feat\.?\s+.+$',     # " feat Artist" at end
        r'\s+ft\.?\s+.+$',       # " ft Artist" at end  
        r'\s+featuring\s+.+$',   # " featuring Artist" at end
        r'\s+with\s+.+$',        # " with Artist" at end
    ]
    
    for pattern in feat_patterns:
        title = re.sub(pattern, '', title, flags=re.IGNORECASE).strip()
    
    # Clean up whitespace and punctuation
    title = re.sub(r'\s+', ' ', title).strip()
    title = re.sub(r'^[-–—:,.\s]+|[-–—:,.\s]+$', '', title).strip()
    
    # If we cleaned too much, return original
    if not title.strip() or len(title.strip()) < 2:
        title = original_title
    
    if title != original_title:
        print(f"🧹 YouTube title cleaned: '{original_title}' → '{title}'")
    
    return title

def clean_youtube_artist(artist_string):
    """
    Clean YouTube artist strings to get primary artist name
    
    Examples:
    'Yung Gravy, bbno$ (BABY GRAVY)' → 'Yung Gravy'
    'Y2K, bbno$' → 'Y2K'
    'LITTLE BIG' → 'LITTLE BIG'
    'Artist "Nickname" Name' → 'Artist Nickname Name'
    'ArtistVEVO' → 'Artist'
    """
    import re
    
    if not artist_string:
        return artist_string
    
    original_artist = artist_string
    
    # Remove all quotes - they're usually not part of artist names
    artist_string = artist_string.replace('"', '').replace("'", '').replace(''', '').replace(''', '').replace('"', '').replace('"', '')
    
    # Remove anything in parentheses (often group/label names)
    artist_string = re.sub(r'\s*\([^)]*\)', '', artist_string).strip()
    
    # Remove anything in brackets (often additional info)
    artist_string = re.sub(r'\s*\[[^\]]*\]', '', artist_string).strip()
    
    # Remove common YouTube channel suffixes
    channel_suffixes = [
        r'\s*VEVO\s*$',
        r'\s*Music\s*$',
        r'\s*Official\s*$',
        r'\s*Records\s*$',
        r'\s*Entertainment\s*$',
        r'\s*TV\s*$',
        r'\s*Channel\s*$'
    ]
    
    for suffix in channel_suffixes:
        artist_string = re.sub(suffix, '', artist_string, flags=re.IGNORECASE).strip()
    
    # Split on common separators and take the first artist
    separators = [',', '&', ' and ', ' x ', ' X ', ' feat.', ' ft.', ' featuring', ' with', ' vs ', ' vs.']
    
    for sep in separators:
        if sep in artist_string:
            parts = artist_string.split(sep)
            artist_string = parts[0].strip()
            break
    
    # Clean up extra whitespace and punctuation
    artist_string = re.sub(r'\s+', ' ', artist_string).strip()
    artist_string = re.sub(r'^\-\s*|\s*\-$', '', artist_string).strip()  # Remove leading/trailing dashes
    artist_string = re.sub(r'^,\s*|\s*,$', '', artist_string).strip()    # Remove leading/trailing commas
    
    # If we cleaned too much, return original
    if not artist_string.strip():
        artist_string = original_artist
    
    if artist_string != original_artist:
        print(f"🧹 YouTube artist cleaned: '{original_artist}' → '{artist_string}'")
    
    return artist_string

def parse_youtube_playlist(url):
    """
    Parse a YouTube Music playlist URL and extract track information using yt-dlp
    Uses flat playlist extraction to avoid rate limits and get all tracks
    Returns a list of track dictionaries compatible with our Track structure
    """
    try:
        # Configure yt-dlp options for flat playlist extraction (avoids rate limits)
        ydl_opts = {
            'quiet': True,
            'no_warnings': True,
            'extract_flat': True,  # Only extract basic info, no individual video metadata
            'flat_playlist': True,  # Extract all playlist entries without hitting API for each video
            'skip_download': True,  # Don't download, just extract IDs and basic info
            # Remove all limits to get complete playlist
        }
        
        tracks = []
        
        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
            # Extract playlist info
            playlist_info = ydl.extract_info(url, download=False)
            
            if not playlist_info:
                print("❌ Could not extract playlist information")
                return None
            
            playlist_name = playlist_info.get('title', 'Unknown Playlist')
            playlist_id = playlist_info.get('id', 'unknown_id')
            entries = playlist_info.get('entries', [])
            
            print(f"🎵 Found YouTube playlist: '{playlist_name}' with {len(entries)} entries")
            
            for entry in entries:
                if not entry:
                    continue
                
                # Extract basic information from flat extraction
                raw_title = entry.get('title', 'Unknown Track')
                raw_uploader = entry.get('uploader', 'Unknown Artist')
                duration = entry.get('duration', 0)
                video_id = entry.get('id', '')
                
                # Clean the track title and artist using our cleaning functions
                cleaned_artist = clean_youtube_artist(raw_uploader)
                cleaned_title = clean_youtube_track_title(raw_title, cleaned_artist)
                
                # Create track object matching GUI structure
                track_data = {
                    'id': video_id,
                    'name': cleaned_title,
                    'artists': [cleaned_artist],
                    'duration_ms': duration * 1000 if duration else 0,
                    'raw_title': raw_title,  # Keep original for reference
                    'raw_artist': raw_uploader,  # Keep original for reference
                    'url': f"https://www.youtube.com/watch?v={video_id}"
                }
                
                tracks.append(track_data)
            
            # Create playlist object matching GUI structure
            playlist_data = {
                'id': playlist_id,
                'name': playlist_name,
                'tracks': tracks,
                'track_count': len(tracks),
                'url': url,
                'source': 'youtube'
            }
            
            print(f"✅ Successfully parsed YouTube playlist: {len(tracks)} tracks extracted")
            return playlist_data
            
    except Exception as e:
        print(f"❌ Error parsing YouTube playlist: {e}")
        return None


# ===================================================================
# FILE ORGANIZATION TEMPLATE ENGINE
# ===================================================================

def _compute_m3u_folder(transfer_dir, context_type, playlist_name, artist_name='', album_name='', year=''):
    """
    Compute the target folder for an M3U file using the template system.

    For playlists: uses playlist_path template, extracts folder portion.
    For albums: uses album_path template, extracts folder portion.

    Returns: absolute folder path
    """
    if context_type == 'album' and artist_name and album_name:
        template_context = {
            'artist': artist_name,
            'albumartist': artist_name,
            'album': album_name,
            'title': 'placeholder',
            'track_number': 1,
            'disc_number': 1,
            'year': year,
            'quality': ''
        }
        folder_path, _ = _get_file_path_from_template(template_context, 'album_path')
        if folder_path:
            return os.path.join(transfer_dir, folder_path)
        # Fallback
        artist_sanitized = _sanitize_filename(artist_name)
        album_sanitized = _sanitize_filename(album_name)
        return os.path.join(transfer_dir, artist_sanitized, f"{artist_sanitized} - {album_sanitized}")
    else:
        template_context = {
            'artist': 'placeholder',
            'albumartist': 'placeholder',
            'album': 'placeholder',
            'title': 'placeholder',
            'playlist_name': playlist_name,
            'track_number': 1,
            'disc_number': 1,
            'year': '',
            'quality': ''
        }
        folder_path, _ = _get_file_path_from_template(template_context, 'playlist_path')
        if folder_path:
            return os.path.join(transfer_dir, folder_path)
        # Fallback
        playlist_sanitized = _sanitize_filename(playlist_name)
        return os.path.join(transfer_dir, playlist_sanitized)


def _build_final_path_for_track(context, spotify_artist, album_info, file_ext):
    """
    SHARED PATH BUILDER - Used by both post-processing AND verification.
    This ensures they always produce the same path.

    Returns: (final_path, folder_created_successfully)
    """
    transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer'))
    track_info = context.get("track_info", {})
    original_search = context.get("original_search_result", {})
    playlist_folder_mode = track_info.get("_playlist_folder_mode", False)

    # Extract year from spotify_album for template use (safe for all modes)
    year = ''  # Empty string instead of 'Unknown' to avoid "Unknown albumName"
    spotify_album = context.get("spotify_album", {})
    if spotify_album and spotify_album.get('release_date'):
        release_date = spotify_album['release_date']
        if release_date and len(release_date) >= 4:
            year = release_date[:4]

    # Determine which template type to use
    if playlist_folder_mode:
        # PLAYLIST MODE
        playlist_name = track_info.get("_playlist_name", "Unknown Playlist")
        track_name = original_search.get('spotify_clean_title') or original_search.get('title', 'Unknown Track')

        template_context = {
            'artist': spotify_artist["name"] if isinstance(spotify_artist, dict) else spotify_artist.name,
            'albumartist': spotify_artist["name"] if isinstance(spotify_artist, dict) else spotify_artist.name,
            'album': track_name,
            'title': track_name,
            'playlist_name': playlist_name,
            'track_number': 1,
            'disc_number': 1,
            'year': year,
            'quality': context.get('_audio_quality', '')
        }

        folder_path, filename_base = _get_file_path_from_template(template_context, 'playlist_path')
        if folder_path and filename_base:
            final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
            os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
            return final_path, True
        else:
            # Fallback
            playlist_name_sanitized = _sanitize_filename(playlist_name)
            playlist_dir = os.path.join(transfer_dir, playlist_name_sanitized)
            os.makedirs(playlist_dir, exist_ok=True)
            artist_name_sanitized = _sanitize_filename(template_context['artist'])
            track_name_sanitized = _sanitize_filename(track_name)
            new_filename = f"{artist_name_sanitized} - {track_name_sanitized}{file_ext}"
            return os.path.join(playlist_dir, new_filename), True

    elif album_info and album_info.get('is_album'):
        # ALBUM MODE
        clean_track_name = album_info.get('clean_track_name', 'Unknown Track')
        if original_search.get('spotify_clean_title'):
            clean_track_name = original_search['spotify_clean_title']
        elif album_info.get('clean_track_name'):
            clean_track_name = album_info['clean_track_name']
        else:
            clean_track_name = original_search.get('title', 'Unknown Track')

        track_number = album_info.get('track_number', 1)
        if track_number is None or not isinstance(track_number, int) or track_number < 1:
            track_number = 1

        # Multi-disc album subfolder support
        disc_number = album_info.get('disc_number', 1)

        template_context = {
            'artist': spotify_artist["name"] if isinstance(spotify_artist, dict) else spotify_artist.name,
            'albumartist': spotify_artist["name"] if isinstance(spotify_artist, dict) else spotify_artist.name,
            'album': album_info['album_name'],
            'title': clean_track_name,
            'track_number': track_number,
            'disc_number': disc_number,
            'year': year,
            'quality': context.get('_audio_quality', '')
        }
        spotify_album = context.get('spotify_album', {})
        total_discs = spotify_album.get('total_discs', 1) if spotify_album else 1

        # Check if user controls disc structure via $disc in their template
        album_template = config_manager.get('file_organization.templates.album_path', '')
        user_controls_disc = '$disc' in album_template

        disc_label = config_manager.get('file_organization.disc_label', 'Disc')

        folder_path, filename_base = _get_file_path_from_template(template_context, 'album_path')
        if folder_path and filename_base:
            if total_discs > 1 and not user_controls_disc:
                disc_folder = f"{disc_label} {disc_number}"
                final_path = os.path.join(transfer_dir, folder_path, disc_folder, filename_base + file_ext)
                os.makedirs(os.path.join(transfer_dir, folder_path, disc_folder), exist_ok=True)
            else:
                final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
                os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
            return final_path, True
        else:
            # Fallback
            artist_name_sanitized = _sanitize_filename(template_context['artist'])
            album_name_sanitized = _sanitize_filename(album_info['album_name'])
            artist_dir = os.path.join(transfer_dir, artist_name_sanitized)
            album_folder_name = f"{artist_name_sanitized} - {album_name_sanitized}"
            album_dir = os.path.join(artist_dir, album_folder_name)
            if total_discs > 1:
                album_dir = os.path.join(album_dir, f"{disc_label} {disc_number}")
            os.makedirs(album_dir, exist_ok=True)
            final_track_name_sanitized = _sanitize_filename(clean_track_name)
            new_filename = f"{track_number:02d} - {final_track_name_sanitized}{file_ext}"
            return os.path.join(album_dir, new_filename), True

    else:
        # SINGLE MODE
        clean_track_name = album_info.get('clean_track_name', 'Unknown Track') if album_info else 'Unknown Track'
        if original_search.get('spotify_clean_title'):
            clean_track_name = original_search['spotify_clean_title']
        elif album_info and album_info.get('clean_track_name'):
            clean_track_name = album_info['clean_track_name']
        else:
            clean_track_name = original_search.get('title', 'Unknown Track')

        template_context = {
            'artist': spotify_artist["name"] if isinstance(spotify_artist, dict) else spotify_artist.name,
            'albumartist': spotify_artist["name"] if isinstance(spotify_artist, dict) else spotify_artist.name,
            'album': album_info.get('album_name', clean_track_name) if album_info else clean_track_name,
            'title': clean_track_name,
            'track_number': 1,
            'disc_number': 1,
            'year': year,
            'quality': context.get('_audio_quality', '')
        }

        folder_path, filename_base = _get_file_path_from_template(template_context, 'single_path')
        if folder_path and filename_base:
            final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
            os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
            return final_path, True
        else:
            # Fallback
            artist_name_sanitized = _sanitize_filename(template_context['artist'])
            final_track_name_sanitized = _sanitize_filename(clean_track_name)
            artist_dir = os.path.join(transfer_dir, artist_name_sanitized)
            single_folder_name = f"{artist_name_sanitized} - {final_track_name_sanitized}"
            single_dir = os.path.join(artist_dir, single_folder_name)
            os.makedirs(single_dir, exist_ok=True)
            new_filename = f"{final_track_name_sanitized}{file_ext}"
            return os.path.join(single_dir, new_filename), True

def _get_audio_quality_string(file_path):
    """
    Read audio file and return a quality descriptor string.

    Returns strings like 'FLAC 16bit', 'MP3-320', 'M4A-256', 'OGG-192'.
    Returns empty string on any error.
    """
    try:
        ext = os.path.splitext(file_path)[1].lower()

        if ext == '.flac':
            audio = FLAC(file_path)
            bits = audio.info.bits_per_sample
            return f"FLAC {bits}bit"

        elif ext == '.mp3':
            from mutagen.mp3 import MP3, BitrateMode
            audio = MP3(file_path)
            bitrate_kbps = audio.info.bitrate // 1000
            if audio.info.bitrate_mode == BitrateMode.VBR:
                return "MP3-VBR"
            return f"MP3-{bitrate_kbps}"

        elif ext in ('.m4a', '.aac', '.mp4'):
            audio = MP4(file_path)
            bitrate_kbps = audio.info.bitrate // 1000
            return f"M4A-{bitrate_kbps}"

        elif ext == '.ogg':
            audio = OggVorbis(file_path)
            bitrate_kbps = audio.info.bitrate // 1000
            return f"OGG-{bitrate_kbps}"

        elif ext == '.opus':
            from mutagen.oggopus import OggOpus
            audio = OggOpus(file_path)
            bitrate_kbps = audio.info.bitrate // 1000
            return f"OPUS-{bitrate_kbps}"

        return ''
    except Exception as e:
        logger.debug(f"Could not determine audio quality for {file_path}: {e}")
        return ''

def _create_lossy_copy(final_path):
    """Convert a FLAC file to MP3 at the user's configured bitrate.

    Only runs when lossy_copy is enabled and the file is a FLAC.
    Places the MP3 alongside the FLAC with the same basename.

    Returns the MP3 path if Blasphemy Mode deleted the original, else None.
    """
    if not config_manager.get('lossy_copy.enabled', False):
        return None

    ext = os.path.splitext(final_path)[1].lower()
    if ext != '.flac':
        return None

    bitrate = config_manager.get('lossy_copy.bitrate', '320')
    mp3_quality = f'MP3-{bitrate}'
    mp3_path = os.path.splitext(final_path)[0] + '.mp3'

    # If $quality was used in filename, swap FLAC quality for MP3 quality
    original_quality = _get_audio_quality_string(final_path)
    if original_quality:
        mp3_basename = os.path.basename(mp3_path)
        if original_quality in mp3_basename:
            mp3_basename = mp3_basename.replace(original_quality, mp3_quality)
            mp3_path = os.path.join(os.path.dirname(mp3_path), mp3_basename)

    ffmpeg_bin = shutil.which('ffmpeg')
    if not ffmpeg_bin:
        local = os.path.join(os.path.dirname(__file__), 'tools', 'ffmpeg')
        if os.path.isfile(local):
            ffmpeg_bin = local
        else:
            print("⚠️ [Lossy Copy] ffmpeg not found — skipping MP3 conversion")
            return None

    try:
        print(f"🎵 [Lossy Copy] Converting to MP3-{bitrate}: {os.path.basename(final_path)}")
        result = subprocess.run([
            ffmpeg_bin, '-i', final_path,
            '-codec:a', 'libmp3lame',
            '-b:a', f'{bitrate}k',
            '-map_metadata', '0',
            '-id3v2_version', '3',
            '-y', mp3_path
        ], capture_output=True, text=True, timeout=120)

        if result.returncode == 0:
            print(f"✅ [Lossy Copy] Created MP3-{bitrate} copy: {os.path.basename(mp3_path)}")
            # Fix QUALITY tag — the FLAC's tag (e.g. "FLAC 24bit") was copied verbatim
            try:
                from mutagen.id3 import ID3, TXXX
                tags = ID3(mp3_path)
                tags.add(TXXX(encoding=3, desc='QUALITY', text=[f'MP3-{bitrate}']))
                tags.save()
            except Exception as tag_err:
                print(f"⚠️ [Lossy Copy] Could not update QUALITY tag: {tag_err}")

            # Blasphemy Mode: delete original FLAC if enabled and MP3 is verified
            if config_manager.get('lossy_copy.delete_original', False):
                try:
                    if os.path.isfile(mp3_path) and os.path.getsize(mp3_path) > 0:
                        from mutagen import File as MutagenFile
                        test_audio = MutagenFile(mp3_path)
                        if test_audio is not None:
                            os.remove(final_path)
                            print(f"🔥 [Blasphemy Mode] Deleted original: {os.path.basename(final_path)}")
                            # Rename LRC file to match the MP3 filename
                            flac_lrc = os.path.splitext(final_path)[0] + '.lrc'
                            if os.path.isfile(flac_lrc):
                                mp3_lrc = os.path.splitext(mp3_path)[0] + '.lrc'
                                try:
                                    os.rename(flac_lrc, mp3_lrc)
                                    print(f"🔥 [Blasphemy Mode] Renamed LRC: {os.path.basename(flac_lrc)} -> {os.path.basename(mp3_lrc)}")
                                except Exception as lrc_err:
                                    print(f"⚠️ [Blasphemy Mode] Could not rename LRC: {lrc_err}")
                            return mp3_path
                        else:
                            print(f"⚠️ [Blasphemy Mode] MP3 failed audio validation, keeping original: {os.path.basename(final_path)}")
                    else:
                        print(f"⚠️ [Blasphemy Mode] MP3 missing or empty, keeping original: {os.path.basename(final_path)}")
                except Exception as del_err:
                    print(f"⚠️ [Blasphemy Mode] Error during original deletion, keeping original: {del_err}")
        else:
            print(f"⚠️ [Lossy Copy] ffmpeg failed: {result.stderr[:200]}")
    except subprocess.TimeoutExpired:
        print(f"⚠️ [Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}")
    except Exception as e:
        print(f"⚠️ [Lossy Copy] Conversion error: {e}")
    return None

def _apply_path_template(template: str, context: dict) -> str:
    """
    Apply template to build file path.

    Args:
        template: Template string like "$artist/$album/$track - $title"
        context: Dict with values like {'artist': 'Drake', 'album': 'Scorpion', ...}

    Returns:
        Processed path string
    """
    # Sanitize context values BEFORE template substitution
    # This prevents '/' in metadata from creating unintended path components
    clean_context = _sanitize_context_values(context)

    result = template

    # Replace variables in order from longest to shortest to avoid partial replacements
    # (e.g., $albumartist must be replaced before $album to prevent "Scorpionartist" from typo "$albumartis")

    # Longest variables first
    result = result.replace('$albumartist', clean_context.get('albumartist', clean_context.get('artist', 'Unknown Artist')))
    result = result.replace('$playlist', clean_context.get('playlist_name', ''))

    # Medium length variables
    result = result.replace('$artistletter', (clean_context.get('artist', 'U') or 'U')[0].upper())
    result = result.replace('$artist', clean_context.get('artist', 'Unknown Artist'))
    result = result.replace('$album', clean_context.get('album', 'Unknown Album'))
    result = result.replace('$title', clean_context.get('title', 'Unknown Track'))
    result = result.replace('$track', f"{clean_context.get('track_number', 1):02d}")
    result = result.replace('$year', str(clean_context.get('year', '')))  # Empty string instead of 'Unknown'

    # Clean up extra spaces that might result from empty variables
    import re
    result = re.sub(r'\s+', ' ', result)  # Multiple spaces to single space
    result = re.sub(r'\s*-\s*-\s*', ' - ', result)  # Clean up double dashes
    result = result.strip()  # Remove leading/trailing spaces

    return result

def _get_file_path_from_template(context: dict, template_type: str = 'album_path') -> tuple:
    """
    Build complete file path using configured templates.

    Args:
        context: Dict with all track/album metadata
        template_type: 'album_path', 'single_path', 'compilation_path', 'playlist_path'

    Returns:
        (folder_path, filename) tuple
    """
    # Check if template system is enabled
    if not config_manager.get('file_organization.enabled', True):
        # Fallback to hardcoded structure
        return None, None

    # Get template from config
    templates = config_manager.get('file_organization.templates', {})
    template = templates.get(template_type)

    if not template:
        # Fallback templates if config missing
        default_templates = {
            'album_path': '$albumartist/$albumartist - $album/$track - $title',
            'single_path': '$artist/$artist - $title/$title',
            'compilation_path': 'Compilations/$album/$track - $artist - $title',
            'playlist_path': '$playlist/$artist - $title'
        }
        template = default_templates.get(template_type, '$artist/$album/$track - $title')

    # Apply template
    full_path = _apply_path_template(template, context)

    # Split into folder and filename
    path_parts = full_path.split('/')

    # Handle $quality and $disc: only substituted in the filename (last component).
    # In folder components they become empty string to prevent album splits
    # when tracks arrive in mixed qualities or disc numbers in folder names.
    import re
    quality_value = context.get('quality', '')
    disc_value = f"{context.get('disc_number', 1):02d}"

    if len(path_parts) > 1:
        folder_parts = path_parts[:-1]
        filename_base = path_parts[-1]

        # Strip $quality and $disc from folder parts and clean up artifacts
        cleaned_folders = []
        for part in folder_parts:
            part = part.replace('$quality', '')
            part = part.replace('$disc', '')
            part = re.sub(r'\s*\[\s*\]', '', part)   # empty []
            part = re.sub(r'\s*\(\s*\)', '', part)   # empty ()
            part = re.sub(r'\s*\{\s*\}', '', part)   # empty {}
            part = re.sub(r'\s*-\s*$', '', part)     # trailing dash
            part = re.sub(r'^\s*-\s*', '', part)     # leading dash
            part = re.sub(r'\s+', ' ', part).strip()
            if part:
                cleaned_folders.append(part)

        # Substitute $quality and $disc in filename only
        filename_base = filename_base.replace('$quality', quality_value)
        filename_base = filename_base.replace('$disc', disc_value)
        # Clean up empty brackets/parens from any variable that resolved to empty
        filename_base = re.sub(r'\s*\[\s*\]', '', filename_base)
        filename_base = re.sub(r'\s*\(\s*\)', '', filename_base)
        filename_base = re.sub(r'\s*\{\s*\}', '', filename_base)
        filename_base = re.sub(r'\s*-\s*$', '', filename_base)
        filename_base = re.sub(r'\s+', ' ', filename_base).strip()

        # Sanitize each folder component
        sanitized_folders = [_sanitize_filename(part) for part in cleaned_folders]
        folder_path = os.path.join(*sanitized_folders) if sanitized_folders else ''

        # Sanitize filename
        filename = _sanitize_filename(filename_base)

        return folder_path, filename
    else:
        # Single component, treat as filename — substitute $quality and $disc
        full_path = full_path.replace('$quality', quality_value)
        full_path = full_path.replace('$disc', disc_value)
        full_path = re.sub(r'\s*\[\s*\]', '', full_path)
        full_path = re.sub(r'\s*\(\s*\)', '', full_path)
        full_path = re.sub(r'\s*\{\s*\}', '', full_path)
        full_path = re.sub(r'\s*-\s*$', '', full_path)
        full_path = re.sub(r'\s+', ' ', full_path).strip()
        return '', _sanitize_filename(full_path)

# METADATA & COVER ART HELPERS (Ported from downloads.py)
# ===================================================================
from mutagen import File as MutagenFile
from mutagen.id3 import ID3, TIT2, TPE1, TALB, TDRC, TRCK, TCON, TPE2, TPOS, TXXX, APIC, UFID, TSRC, TBPM
from mutagen.flac import FLAC, Picture
from mutagen.mp4 import MP4, MP4Cover, MP4FreeForm
from mutagen.oggvorbis import OggVorbis
from mutagen.apev2 import APEv2, APENoHeaderError
import urllib.request

def _strip_all_non_audio_tags(file_path: str) -> dict:
    """
    Strip ALL non-audio tag containers from a file before metadata rewriting.
    MP3 files from Soulseek commonly carry APEv2 tags (foobar2000 users)
    with stale metadata that Mutagen's ID3 handler cannot see or clear.
    Must run BEFORE MutagenFile() opens the file.
    """
    summary = {'apev2_stripped': False, 'apev2_tag_count': 0}
    ext = os.path.splitext(file_path)[1].lower()
    if ext != '.mp3':
        return summary
    try:
        apev2_tags = APEv2(file_path)
        tag_count = len(apev2_tags)
        tag_keys = list(apev2_tags.keys())
        apev2_tags.delete(file_path)
        summary['apev2_stripped'] = True
        summary['apev2_tag_count'] = tag_count
        print(f"🧹 Stripped {tag_count} APEv2 tags: {', '.join(tag_keys[:10])}")
    except APENoHeaderError:
        pass  # No APEv2 tags — common case
    except Exception as e:
        print(f"⚠️ Could not strip APEv2 tags (non-fatal): {e}")
    return summary

def _verify_metadata_written(file_path: str) -> bool:
    """Re-open file and verify core metadata fields are present."""
    try:
        check = MutagenFile(file_path)
        if check is None or check.tags is None:
            print(f"❌ [VERIFY] Tags are None after save: {file_path}")
            return False
        title_found = False
        artist_found = False
        if isinstance(check.tags, ID3):
            title_found = bool(check.tags.getall('TIT2'))
            artist_found = bool(check.tags.getall('TPE1'))
            # Confirm APEv2 is gone
            try:
                APEv2(file_path)
                print(f"⚠️ [VERIFY] APEv2 tags still present after processing!")
                return False
            except APENoHeaderError:
                pass
        elif isinstance(check, (FLAC, OggVorbis)):
            title_found = bool(check.get('title'))
            artist_found = bool(check.get('artist'))
        elif isinstance(check, MP4):
            title_found = bool(check.get('\xa9nam'))
            artist_found = bool(check.get('\xa9ART'))
        if not title_found or not artist_found:
            print(f"❌ [VERIFY] Missing metadata - title:{title_found} artist:{artist_found}")
            return False
        print(f"✅ [VERIFY] Metadata verified OK")
        return True
    except Exception as e:
        print(f"⚠️ [VERIFY] Verification error (non-fatal): {e}")
        return False

def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool:
    """
    Core function to enhance audio file metadata using Spotify data.
    Thread-safe with per-file locking to prevent concurrent metadata writes.

    Opens the file once in non-easy mode, clears all tags in memory, writes
    new tags using format-specific frames/keys, embeds album art and source
    IDs, then saves once.  This avoids the old clear→save→reopen pattern
    which stripped the ID3v2 header from MP3 files, leaving them tagless.
    """
    if not config_manager.get('metadata_enhancement.enabled', True):
        print("🎵 Metadata enhancement disabled in config.")
        return True

    # Acquire per-file lock to prevent concurrent metadata writes to the same file
    file_lock = _get_file_lock(file_path)
    with file_lock:
        print(f"🎵 Enhancing metadata for: {os.path.basename(file_path)}")
        try:
            # Strip APEv2 tags from MP3 (invisible to ID3 handler)
            strip_summary = _strip_all_non_audio_tags(file_path)

            audio_file = MutagenFile(file_path)
            if audio_file is None:
                print(f"❌ Could not load audio file with Mutagen: {file_path}")
                return False

            # ── Wipe ALL existing tags in memory (NO save yet) ──
            # Files from Soulseek carry random metadata (wrong comments,
            # encoder info, ReplayGain, old album art, random TXXX frames).
            if hasattr(audio_file, 'clear_pictures'):
                audio_file.clear_pictures()

            if audio_file.tags is not None:
                # Log what's being cleared for debugging
                if len(audio_file.tags) > 0:
                    tag_keys = list(audio_file.tags.keys())[:15]
                    print(f"🧹 Clearing {len(audio_file.tags)} existing tags: "
                          f"{', '.join(str(k) for k in tag_keys)}")
                audio_file.tags.clear()
            else:
                audio_file.add_tags()

            metadata = _extract_spotify_metadata(context, artist, album_info)
            if not metadata:
                print("⚠️ Could not extract Spotify metadata, saving with cleared tags.")
                if isinstance(audio_file.tags, ID3):
                    audio_file.save(v1=0, v2_version=4)
                elif isinstance(audio_file, FLAC):
                    audio_file.save(deleteid3=True)
                else:
                    audio_file.save()
                return True

            # ── Write standard tags using format-specific API ──
            track_num_str = f"{metadata.get('track_number', 1)}/{metadata.get('total_tracks', 1)}"

            if isinstance(audio_file.tags, ID3):
                # MP3: write ID3 frames directly
                if metadata.get('title'):
                    audio_file.tags.add(TIT2(encoding=3, text=[metadata['title']]))
                if metadata.get('artist'):
                    audio_file.tags.add(TPE1(encoding=3, text=[metadata['artist']]))
                if metadata.get('album_artist'):
                    audio_file.tags.add(TPE2(encoding=3, text=[metadata['album_artist']]))
                if metadata.get('album'):
                    audio_file.tags.add(TALB(encoding=3, text=[metadata['album']]))
                if metadata.get('date'):
                    audio_file.tags.add(TDRC(encoding=3, text=[metadata['date']]))
                if metadata.get('genre'):
                    audio_file.tags.add(TCON(encoding=3, text=[metadata['genre']]))
                audio_file.tags.add(TRCK(encoding=3, text=[track_num_str]))
                if metadata.get('disc_number'):
                    audio_file.tags.add(TPOS(encoding=3, text=[str(metadata['disc_number'])]))

            elif isinstance(audio_file, (FLAC, OggVorbis)):
                # FLAC / OGG Vorbis: dict-style VorbisComment tags
                if metadata.get('title'):
                    audio_file['title'] = [metadata['title']]
                if metadata.get('artist'):
                    audio_file['artist'] = [metadata['artist']]
                if metadata.get('album_artist'):
                    audio_file['albumartist'] = [metadata['album_artist']]
                if metadata.get('album'):
                    audio_file['album'] = [metadata['album']]
                if metadata.get('date'):
                    audio_file['date'] = [metadata['date']]
                if metadata.get('genre'):
                    audio_file['genre'] = [metadata['genre']]
                audio_file['tracknumber'] = [track_num_str]
                if metadata.get('disc_number'):
                    audio_file['discnumber'] = [str(metadata['disc_number'])]

            elif isinstance(audio_file, MP4):
                # MP4 / M4A: Apple-style tag keys
                if metadata.get('title'):
                    audio_file['\xa9nam'] = [metadata['title']]
                if metadata.get('artist'):
                    audio_file['\xa9ART'] = [metadata['artist']]
                if metadata.get('album_artist'):
                    audio_file['aART'] = [metadata['album_artist']]
                if metadata.get('album'):
                    audio_file['\xa9alb'] = [metadata['album']]
                if metadata.get('date'):
                    audio_file['\xa9day'] = [metadata['date']]
                if metadata.get('genre'):
                    audio_file['\xa9gen'] = [metadata['genre']]
                track_num = metadata.get('track_number', 1)
                total_tracks = metadata.get('total_tracks', 1)
                audio_file['trkn'] = [(track_num, total_tracks)]
                if metadata.get('disc_number'):
                    audio_file['disk'] = [(metadata['disc_number'], 0)]

            # ── Embed album art on the same object ──
            if config_manager.get('metadata_enhancement.embed_album_art', True):
                _embed_album_art_metadata(audio_file, metadata)

            # ── Embed source IDs (Spotify, MusicBrainz, etc.) on the same object ──
            _embed_source_ids(audio_file, metadata)

            # ── Embed audio quality tag ──
            quality = context.get('_audio_quality', '')
            if quality:
                if isinstance(audio_file.tags, ID3):
                    audio_file.tags.add(TXXX(encoding=3, desc='QUALITY', text=[quality]))
                elif isinstance(audio_file, (FLAC, OggVorbis)):
                    audio_file['quality'] = [quality]
                elif isinstance(audio_file, MP4):
                    audio_file['----:com.apple.iTunes:QUALITY'] = [MP4FreeForm(quality.encode('utf-8'))]

            # ── Single save for everything ──
            if isinstance(audio_file.tags, ID3):
                audio_file.save(v1=0, v2_version=4)
            elif isinstance(audio_file, FLAC):
                audio_file.save(deleteid3=True)
            else:
                audio_file.save()

            # Verify metadata was written
            verified = _verify_metadata_written(file_path)
            if verified:
                print("✅ Metadata enhanced successfully.")
            else:
                print("⚠️ Metadata saved but verification found issues (see above).")
            return True
        except Exception as e:
            import traceback
            print(f"❌ Error enhancing metadata for {file_path}: {e}")
            print(f"❌ [Metadata Debug] Exception type: {type(e).__name__}")
            print(f"❌ [Metadata Debug] File exists: {os.path.exists(file_path)}")
            print(f"❌ [Metadata Debug] Artist: {artist.get('name', 'MISSING') if artist else 'None'}")
            print(f"❌ [Metadata Debug] Album info: {album_info.get('album_name', 'MISSING') if album_info else 'None'}")
            print(f"❌ [Metadata Debug] Traceback:\n{traceback.format_exc()}")
            return False

def _generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool:
    """
    Generate LRC lyrics file using LRClib API.
    Elegant addition to post-processing - extracts metadata from existing context.
    """
    try:
        # Extract track information from existing context (same as metadata enhancement)
        original_search = context.get("original_search_result", {})
        spotify_album = context.get("spotify_album")

        # Get track metadata
        track_name = (original_search.get('spotify_clean_title') or
                     original_search.get('title', 'Unknown Track'))

        # Handle artist parameter (can be dict or object)
        if isinstance(artist, dict):
            artist_name = artist.get('name', 'Unknown Artist')
        elif hasattr(artist, 'name'):
            artist_name = artist.name
        else:
            artist_name = str(artist) if artist else 'Unknown Artist'
        album_name = None
        duration_seconds = None

        # Get album name if available
        if album_info.get('is_album'):
            album_name = (original_search.get('spotify_clean_album') or
                         album_info.get('album_name') or
                         (spotify_album.get('name') if spotify_album else None))

        # Get duration from original search context
        if original_search.get('duration_ms'):
            duration_seconds = int(original_search['duration_ms'] / 1000)

        # Generate LRC file using lyrics client
        success = lyrics_client.create_lrc_file(
            audio_file_path=file_path,
            track_name=track_name,
            artist_name=artist_name,
            album_name=album_name,
            duration_seconds=duration_seconds
        )

        if success:
            print(f"🎵 LRC file generated for: {track_name}")
        else:
            print(f"🎵 No lyrics found for: {track_name}")

        return success

    except Exception as e:
        print(f"❌ Error generating LRC file for {file_path}: {e}")
        return False

def _extract_spotify_metadata(context: dict, artist: dict, album_info: dict) -> dict:
    """Extracts a comprehensive metadata dictionary from the provided context."""
    metadata = {}
    original_search = context.get("original_search_result", {})
    spotify_album = context.get("spotify_album")

    # Priority 1: Spotify clean title from context
    if original_search.get('spotify_clean_title'):
        metadata['title'] = original_search['spotify_clean_title']
        print(f"🎵 Metadata: Using Spotify clean title: '{metadata['title']}'")
    # Priority 2: Album info clean name
    elif album_info.get('clean_track_name'):
        metadata['title'] = album_info['clean_track_name']
        print(f"🎵 Metadata: Using album info clean name: '{metadata['title']}'")
    # Priority 3: Original title as fallback
    else:
        metadata['title'] = original_search.get('title', '')
        print(f"🎵 Metadata: Using original title as fallback: '{metadata['title']}'")
    # Handle multiple artists from Spotify data
    original_search = context.get("original_search_result", {})
    if 'artists' in original_search and isinstance(original_search['artists'], list) and len(original_search['artists']) > 0:
        # Join all artists with semicolon separator (standard format)
        all_artists = []
        for a in original_search['artists']:
            if isinstance(a, dict) and 'name' in a:
                all_artists.append(a['name'])
            elif isinstance(a, str):
                all_artists.append(a)
            else:
                all_artists.append(str(a))
        metadata['artist'] = ', '.join(all_artists)
        print(f"🎵 Metadata: Using all artists: '{metadata['artist']}'")
    else:
        # Fallback to single artist
        metadata['artist'] = artist.get('name', '')
        print(f"🎵 Metadata: Using primary artist: '{metadata['artist']}'")

    metadata['album_artist'] = artist.get('name', '') # Crucial for library organization

    if album_info.get('is_album'):
        metadata['album'] = album_info.get('album_name', 'Unknown Album')
        track_num = album_info.get('track_number', 1)
        metadata['track_number'] = track_num
        metadata['total_tracks'] = spotify_album.get('total_tracks', 1) if spotify_album else 1
        print(f"🎵 [METADATA] Album track - track_number: {track_num}, album: {metadata['album']}")
    else:
        # SAFEGUARD: If we have spotify_album context, never use track title as album name
        # This prevents album tracks from being tagged as singles due to classification errors
        if spotify_album and spotify_album.get('name'):
            print(f"🛡️ [SAFEGUARD] Using spotify_album name instead of track title for album metadata")
            metadata['album'] = spotify_album['name']
            # Use corrected track_number from album_info (which should be updated by post-processing)
            corrected_track_number = album_info.get('track_number', 1) if album_info else 1
            metadata['track_number'] = corrected_track_number
            metadata['total_tracks'] = spotify_album.get('total_tracks', 1)
            print(f"🛡️ [SAFEGUARD] Using track_number: {corrected_track_number}")
        else:
            metadata['album'] = metadata['title'] # For true singles, album is the title
            metadata['track_number'] = 1
            metadata['total_tracks'] = 1

    # Always write disc_number to overwrite any stale tags from the soulseek source.
    # Without this, original disc tags persist and can cause media servers (Plex) to
    # split a single album into standard/deluxe based on differing disc numbers.
    # Priority: original_search context (from API) > album_info > default to 1
    disc_num = original_search.get('disc_number')
    if disc_num is None and album_info:
        disc_num = album_info.get('disc_number')
    if disc_num is None:
        disc_num = 1
    metadata['disc_number'] = disc_num

    if spotify_album and spotify_album.get('release_date'):
        metadata['date'] = spotify_album['release_date'][:4]

    if artist.get('genres'):
        metadata['genre'] = ', '.join(artist['genres'][:2])

    metadata['album_art_url'] = album_info.get('album_image_url')

    # Extract source IDs (Spotify or iTunes) for tag embedding
    track_info = context.get("track_info", {})
    if track_info and track_info.get('id'):
        # Spotify track IDs are alphanumeric strings; iTunes IDs are numeric
        track_id = str(track_info['id'])
        if track_id.isdigit():
            metadata['itunes_track_id'] = track_id
        else:
            metadata['spotify_track_id'] = track_id
    if artist.get('id'):
        artist_id = str(artist['id'])
        if artist_id.isdigit():
            metadata['itunes_artist_id'] = artist_id
        else:
            metadata['spotify_artist_id'] = artist_id
    if spotify_album and spotify_album.get('id'):
        album_id = str(spotify_album['id'])
        if album_id.isdigit():
            metadata['itunes_album_id'] = album_id
        else:
            metadata['spotify_album_id'] = album_id

    # Summary log for debugging metadata issues (e.g. wrong album_artist / track_number)
    print(f"📋 [Metadata Summary] title='{metadata.get('title')}' | artist='{metadata.get('artist')}' | album_artist='{metadata.get('album_artist')}' | album='{metadata.get('album')}' | track={metadata.get('track_number')}/{metadata.get('total_tracks')} | disc={metadata.get('disc_number')}")

    return metadata

def _embed_album_art_metadata(audio_file, metadata: dict):
    """Downloads and embeds high-quality Spotify album art into the file."""
    try:
        art_url = metadata.get('album_art_url')
        if not art_url:
            print("🎨 No album art URL available for embedding.")
            return

        with urllib.request.urlopen(art_url, timeout=10) as response:
            image_data = response.read()
            mime_type = response.info().get_content_type()

        if not image_data:
            print("❌ Failed to download album art data.")
            return

        # MP3 (ID3)
        if isinstance(audio_file.tags, ID3):
            audio_file.tags.add(APIC(encoding=3, mime=mime_type, type=3, desc='Cover', data=image_data))
        # FLAC
        elif isinstance(audio_file, FLAC):
            picture = Picture()
            picture.data = image_data
            picture.type = 3
            picture.mime = mime_type
            picture.width = 640
            picture.height = 640
            picture.depth = 24
            audio_file.add_picture(picture)
        # MP4/M4A
        elif isinstance(audio_file, MP4):
            fmt = MP4Cover.FORMAT_JPEG if 'jpeg' in mime_type else MP4Cover.FORMAT_PNG
            audio_file['covr'] = [MP4Cover(image_data, imageformat=fmt)]
        
        print("🎨 Album art successfully embedded.")
    except Exception as e:
        print(f"❌ Error embedding album art: {e}")

def _embed_source_ids(audio_file, metadata: dict):
    """
    Lookup MusicBrainz, Deezer, and AudioDB metadata, then embed them along
    with Spotify/iTunes source IDs as custom tags into the audio file.
    Tags written: source IDs, BPM (Deezer), mood/style (AudioDB), ISRC
    (MB→Deezer fallback), and merged genres (Spotify+MB+AudioDB).
    One file write, one shot.  Concurrent calls are safe — each service has
    its own global rate limiter.
    Operates on a non-easy-mode MutagenFile object (caller must save).
    """
    try:
        # ── Helper: normalize + compare names (same logic as enrichment workers) ──
        from difflib import SequenceMatcher
        def _names_match(a: str, b: str, threshold: float = 0.75) -> bool:
            if not a or not b:
                return False
            norm = lambda s: re.sub(r'[^a-z0-9 ]', '', re.sub(r'\(.*?\)', '', s).lower()).strip()
            return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold

        # ── 1. Collect Spotify / iTunes IDs already in metadata ──
        id_tags = {}
        if metadata.get('spotify_track_id'):
            id_tags['SPOTIFY_TRACK_ID'] = metadata['spotify_track_id']
        if metadata.get('spotify_artist_id'):
            id_tags['SPOTIFY_ARTIST_ID'] = metadata['spotify_artist_id']
        if metadata.get('spotify_album_id'):
            id_tags['SPOTIFY_ALBUM_ID'] = metadata['spotify_album_id']
        if metadata.get('itunes_track_id'):
            id_tags['ITUNES_TRACK_ID'] = metadata['itunes_track_id']
        if metadata.get('itunes_artist_id'):
            id_tags['ITUNES_ARTIST_ID'] = metadata['itunes_artist_id']
        if metadata.get('itunes_album_id'):
            id_tags['ITUNES_ALBUM_ID'] = metadata['itunes_album_id']

        # ── 2a. MusicBrainz lookup for MBID, genres, and ISRC ──
        # The global rate limiter in musicbrainz_client.py serializes all API
        # calls (worker + any number of post-processing threads) to 1 req/sec
        # via _api_call_lock, so no pause/resume needed.
        recording_mbid = None
        artist_mbid = None
        mb_genres = []
        isrc = None
        track_title = metadata.get('title', '')
        # Use album_artist (single primary artist) for MB lookup, not the
        # comma-joined multi-artist field which would give bad search results
        artist_name = metadata.get('album_artist', '') or metadata.get('artist', '')

        if not config_manager.get('musicbrainz.embed_tags', True):
            # Skip MB lookup, just write Spotify/iTunes IDs if any
            pass
        elif track_title and artist_name:
            try:
                mb_service = mb_worker.mb_service if mb_worker else None
                if mb_service:
                    result = mb_service.match_recording(track_title, artist_name)
                    if result and result.get('mbid'):
                        recording_mbid = result['mbid']
                        id_tags['MUSICBRAINZ_RECORDING_ID'] = recording_mbid
                        print(f"🎵 MusicBrainz recording matched: {recording_mbid}")

                        # Lookup recording details for ISRC and genres
                        details = mb_service.mb_client.get_recording(
                            recording_mbid, includes=['isrcs', 'genres']
                        )
                        if details:
                            isrcs = details.get('isrcs', [])
                            if isrcs:
                                isrc = isrcs[0]
                            mb_genres = [
                                g['name'] for g in sorted(
                                    details.get('genres', []),
                                    key=lambda x: x.get('count', 0),
                                    reverse=True
                                )
                            ]

                    # Use track artist (not album artist) for artist MBID tag
                    track_artist_name = metadata.get('artist', '') or artist_name
                    # For multi-artist tracks, use the first artist only
                    if ', ' in track_artist_name:
                        track_artist_name = track_artist_name.split(', ')[0]
                    artist_result = mb_service.match_artist(track_artist_name)
                    if artist_result and artist_result.get('mbid'):
                        artist_mbid = artist_result['mbid']
                        id_tags['MUSICBRAINZ_ARTIST_ID'] = artist_mbid

                    # Get release (album) MBID via thread-safe in-memory cache.
                    # Without this, concurrent threads each call match_release
                    # and get different release variants, splitting albums.
                    album_name_for_mb = metadata.get('album', '')
                    if album_name_for_mb:
                        _rc_key = (album_name_for_mb.lower().strip(), artist_name.lower().strip())
                        with _mb_release_cache_lock:
                            if _rc_key in _mb_release_cache:
                                _rc_mbid = _mb_release_cache[_rc_key]
                            else:
                                try:
                                    _rc_result = mb_service.match_release(album_name_for_mb, artist_name)
                                    _rc_mbid = _rc_result.get('mbid', '') if _rc_result else ''
                                except Exception:
                                    _rc_mbid = ''
                                _mb_release_cache[_rc_key] = _rc_mbid
                        if _rc_mbid:
                            id_tags['MUSICBRAINZ_RELEASE_ID'] = _rc_mbid
                else:
                    print("⚠️ MusicBrainz worker not available, skipping MBID lookup")
            except Exception as e:
                print(f"⚠️ MusicBrainz lookup failed (non-fatal): {e}")

        # ── 2b. Deezer lookup for BPM, ISRC fallback, and source IDs ──
        deezer_bpm = None
        deezer_isrc = None
        if not config_manager.get('deezer.embed_tags', True):
            pass
        elif track_title and artist_name:
            try:
                dz_client = deezer_worker.client if deezer_worker else None
                if dz_client:
                    dz_result = dz_client.search_track(artist_name, track_title)
                    if dz_result and _names_match(dz_result.get('title', ''), track_title) and \
                       _names_match(dz_result.get('artist', {}).get('name', ''), artist_name):
                        dz_track_id = dz_result['id']
                        id_tags['DEEZER_TRACK_ID'] = str(dz_track_id)
                        dz_artist_id = dz_result.get('artist', {}).get('id')
                        if dz_artist_id:
                            id_tags['DEEZER_ARTIST_ID'] = str(dz_artist_id)
                        print(f"🎵 Deezer track matched: {dz_track_id}")

                        # Get full track details for BPM and ISRC
                        dz_details = dz_client.get_track(dz_track_id)
                        if dz_details:
                            bpm_val = dz_details.get('bpm')
                            if bpm_val and bpm_val > 0:
                                deezer_bpm = bpm_val
                            dz_isrc = dz_details.get('isrc')
                            if dz_isrc:
                                deezer_isrc = dz_isrc
                else:
                    print("⚠️ Deezer worker not available, skipping Deezer lookup")
            except Exception as e:
                print(f"⚠️ Deezer lookup failed (non-fatal): {e}")

        # ── 2c. AudioDB lookup for mood, style, genre, and source ID ──
        audiodb_mood = None
        audiodb_style = None
        audiodb_genre = None
        if not config_manager.get('audiodb.embed_tags', True):
            pass
        elif track_title and artist_name:
            try:
                adb_client = audiodb_worker.client if audiodb_worker else None
                if adb_client:
                    adb_result = adb_client.search_track(artist_name, track_title)
                    if adb_result and _names_match(adb_result.get('strTrack', ''), track_title) and \
                       _names_match(adb_result.get('strArtist', ''), artist_name):
                        adb_track_id = adb_result.get('idTrack')
                        if adb_track_id:
                            id_tags['AUDIODB_TRACK_ID'] = str(adb_track_id)
                            print(f"🎵 AudioDB track matched: {adb_track_id}")
                        # Use AudioDB's MusicBrainz IDs as fallbacks for any missing from MB lookup
                        adb_mb_track = adb_result.get('strMusicBrainzID')
                        if adb_mb_track and 'MUSICBRAINZ_RECORDING_ID' not in id_tags:
                            id_tags['MUSICBRAINZ_RECORDING_ID'] = adb_mb_track
                            recording_mbid = adb_mb_track
                            print(f"🎵 MusicBrainz recording ID from AudioDB fallback: {adb_mb_track}")
                        # NOTE: AudioDB's strMusicBrainzAlbumID is intentionally
                        # NOT used as a fallback for MUSICBRAINZ_RELEASE_ID.
                        # AudioDB links each track to its original album in MB,
                        # which differs per track on compilations and splits
                        # albums in players like Navidrome. Album MBID must come
                        # from match_release (cached) to stay consistent.
                        adb_mb_artist = adb_result.get('strMusicBrainzArtistID')
                        if adb_mb_artist and 'MUSICBRAINZ_ARTIST_ID' not in id_tags:
                            id_tags['MUSICBRAINZ_ARTIST_ID'] = adb_mb_artist
                            artist_mbid = adb_mb_artist
                            print(f"🎵 MusicBrainz artist ID from AudioDB fallback: {adb_mb_artist}")
                        audiodb_mood = adb_result.get('strMood') or None
                        audiodb_style = adb_result.get('strStyle') or None
                        audiodb_genre = adb_result.get('strGenre') or None
                else:
                    print("⚠️ AudioDB worker not available, skipping AudioDB lookup")
            except Exception as e:
                print(f"⚠️ AudioDB lookup failed (non-fatal): {e}")

        if not id_tags and not deezer_bpm and not deezer_isrc and not audiodb_mood and not audiodb_style:
            return

        # ── 3. Write all tags into the file ──
        written = []

        # MP3 (ID3)
        if isinstance(audio_file.tags, ID3):
            for tag_name, value in id_tags.items():
                if tag_name == 'MUSICBRAINZ_RECORDING_ID':
                    audio_file.tags.add(UFID(owner='http://musicbrainz.org', data=value.encode('ascii')))
                    written.append('UFID:http://musicbrainz.org')
                elif tag_name == 'MUSICBRAINZ_ARTIST_ID':
                    audio_file.tags.add(TXXX(encoding=3, desc='MusicBrainz Artist Id', text=[value]))
                    written.append('TXXX:MusicBrainz Artist Id')
                elif tag_name == 'MUSICBRAINZ_RELEASE_ID':
                    audio_file.tags.add(TXXX(encoding=3, desc='MusicBrainz Album Id', text=[value]))
                    written.append('TXXX:MusicBrainz Album Id')
                else:
                    audio_file.tags.add(TXXX(encoding=3, desc=tag_name, text=[str(value)]))
                    written.append(f'TXXX:{tag_name}')

        # FLAC / OGG Vorbis
        elif isinstance(audio_file, (FLAC, OggVorbis)):
            for tag_name, value in id_tags.items():
                if tag_name == 'MUSICBRAINZ_RECORDING_ID':
                    audio_file['MUSICBRAINZ_TRACKID'] = [value]
                    written.append('MUSICBRAINZ_TRACKID')
                elif tag_name == 'MUSICBRAINZ_ARTIST_ID':
                    audio_file['MUSICBRAINZ_ARTISTID'] = [value]
                    written.append('MUSICBRAINZ_ARTISTID')
                elif tag_name == 'MUSICBRAINZ_RELEASE_ID':
                    audio_file['MUSICBRAINZ_ALBUMID'] = [value]
                    written.append('MUSICBRAINZ_ALBUMID')
                else:
                    audio_file[tag_name] = [str(value)]
                    written.append(tag_name)

        # MP4 (M4A/AAC)
        elif isinstance(audio_file, MP4):
            for tag_name, value in id_tags.items():
                if tag_name == 'MUSICBRAINZ_RECORDING_ID':
                    key = '----:com.apple.iTunes:MusicBrainz Track Id'
                elif tag_name == 'MUSICBRAINZ_ARTIST_ID':
                    key = '----:com.apple.iTunes:MusicBrainz Artist Id'
                elif tag_name == 'MUSICBRAINZ_RELEASE_ID':
                    key = '----:com.apple.iTunes:MusicBrainz Album Id'
                else:
                    key = f'----:com.apple.iTunes:{tag_name}'
                audio_file[key] = [MP4FreeForm(str(value).encode('utf-8'))]
                written.append(key)

        if written:
            print(f"🔗 Embedded IDs: {', '.join(written)}")

        # ── 3b. Write BPM tag (from Deezer) ──
        if deezer_bpm and deezer_bpm > 0:
            bpm_int = int(deezer_bpm)
            if isinstance(audio_file.tags, ID3):
                audio_file.tags.add(TBPM(encoding=3, text=[str(bpm_int)]))
            elif isinstance(audio_file, (FLAC, OggVorbis)):
                audio_file['BPM'] = [str(bpm_int)]
            elif isinstance(audio_file, MP4):
                audio_file['tmpo'] = [bpm_int]
            print(f"🥁 BPM: {bpm_int}")

        # ── 3c. Write mood tag (from AudioDB) ──
        if audiodb_mood:
            if isinstance(audio_file.tags, ID3):
                audio_file.tags.add(TXXX(encoding=3, desc='MOOD', text=[audiodb_mood]))
            elif isinstance(audio_file, (FLAC, OggVorbis)):
                audio_file['MOOD'] = [audiodb_mood]
            elif isinstance(audio_file, MP4):
                audio_file['----:com.apple.iTunes:MOOD'] = [MP4FreeForm(audiodb_mood.encode('utf-8'))]
            print(f"🎭 Mood: {audiodb_mood}")

        # ── 3d. Write style tag (from AudioDB) ──
        if audiodb_style:
            if isinstance(audio_file.tags, ID3):
                audio_file.tags.add(TXXX(encoding=3, desc='STYLE', text=[audiodb_style]))
            elif isinstance(audio_file, (FLAC, OggVorbis)):
                audio_file['STYLE'] = [audiodb_style]
            elif isinstance(audio_file, MP4):
                audio_file['----:com.apple.iTunes:STYLE'] = [MP4FreeForm(audiodb_style.encode('utf-8'))]
            print(f"🎨 Style: {audiodb_style}")

        # ── 4. Merge genres (Spotify + MusicBrainz + AudioDB) and overwrite tag ──
        enrichment_genres = mb_genres + ([audiodb_genre] if audiodb_genre else [])
        if enrichment_genres:
            spotify_genres = [g.strip() for g in metadata.get('genre', '').split(',') if g.strip()]
            seen = set()
            merged = []
            for g in spotify_genres + enrichment_genres:
                key = g.strip().lower()
                if key and key not in seen:
                    seen.add(key)
                    merged.append(g.strip().title())
                if len(merged) >= 5:
                    break

            if merged:
                genre_string = ', '.join(merged)
                if isinstance(audio_file.tags, ID3):
                    audio_file.tags.add(TCON(encoding=3, text=[genre_string]))
                elif isinstance(audio_file, (FLAC, OggVorbis)):
                    audio_file['GENRE'] = [genre_string]
                elif isinstance(audio_file, MP4):
                    audio_file['\xa9gen'] = [genre_string]
                print(f"🎶 Genres merged: {genre_string}")

        # ── 5. Write ISRC if available (MusicBrainz first, Deezer fallback) ──
        final_isrc = isrc or deezer_isrc
        if final_isrc:
            if isinstance(audio_file.tags, ID3):
                audio_file.tags.add(TSRC(encoding=3, text=[final_isrc]))
            elif isinstance(audio_file, (FLAC, OggVorbis)):
                audio_file['ISRC'] = [final_isrc]
            elif isinstance(audio_file, MP4):
                audio_file['----:com.apple.iTunes:ISRC'] = [MP4FreeForm(final_isrc.encode('utf-8'))]
            source = "MusicBrainz" if isrc else "Deezer"
            print(f"🔖 ISRC ({source}): {final_isrc}")

    except Exception as e:
        print(f"⚠️ Error embedding source IDs (non-fatal): {e}")

def _download_cover_art(album_info: dict, target_dir: str):
    """Downloads cover.jpg into the specified directory."""
    try:
        cover_path = os.path.join(target_dir, "cover.jpg")
        if os.path.exists(cover_path):
            return

        art_url = album_info.get('album_image_url')
        if not art_url:
            print("📷 No cover art URL available for download.")
            return

        with urllib.request.urlopen(art_url, timeout=10) as response:
            image_data = response.read()
        
        with open(cover_path, 'wb') as f:
            f.write(image_data)
        
        print(f"✅ Cover art downloaded to: {cover_path}")
    except Exception as e:
        print(f"❌ Error downloading cover.jpg: {e}")




def _get_spotify_album_tracks(spotify_album: dict) -> list:
    """Fetches all tracks for a given Spotify album ID."""
    if not spotify_album or not spotify_album.get('id'):
        return []
    try:
        tracks_data = spotify_client.get_album_tracks(spotify_album['id'])
        if tracks_data and 'items' in tracks_data:
            return [{
                'name': item.get('name'),
                'track_number': item.get('track_number'),
                'disc_number': item.get('disc_number', 1),
                'id': item.get('id')
            } for item in tracks_data['items']]
        return []
    except Exception as e:
        print(f"❌ Error fetching Spotify album tracks: {e}")
        return []

def _match_track_to_spotify_title(slsk_track_meta: dict, spotify_tracks: list) -> dict:
    """
    Intelligently matches a Soulseek track to a track from the official Spotify
    tracklist using track numbers and title similarity. Returns the matched Spotify track data.
    """
    if not spotify_tracks:
        return slsk_track_meta # Return original if no list to match against

    # Priority 1: Match by track number
    if slsk_track_meta.get('track_number'):
        track_num = slsk_track_meta['track_number']
        for sp_track in spotify_tracks:
            if sp_track.get('track_number') == track_num:
                print(f"✅ Matched track by number ({track_num}): '{slsk_track_meta['title']}' -> '{sp_track['name']}'")
                # Return a new dict with the corrected title and number
                return {
                    'title': sp_track['name'],
                    'artist': slsk_track_meta.get('artist'),
                    'album': slsk_track_meta.get('album'),
                    'track_number': sp_track['track_number'],
                    'disc_number': sp_track.get('disc_number', 1)
                }

    # Priority 2: Match by title similarity (if track number fails)
    best_match = None
    best_score = 0.6 # Require a decent similarity
    for sp_track in spotify_tracks:
        score = matching_engine.similarity_score(
            matching_engine.normalize_string(slsk_track_meta.get('title', '')),
            matching_engine.normalize_string(sp_track.get('name', ''))
        )
        if score > best_score:
            best_score = score
            best_match = sp_track
    
    if best_match:
        print(f"✅ Matched track by title similarity ({best_score:.2f}): '{slsk_track_meta['title']}' -> '{best_match['name']}'")
        return {
            'title': best_match['name'],
            'artist': slsk_track_meta.get('artist'),
            'album': slsk_track_meta.get('album'),
            'track_number': best_match['track_number'],
            'disc_number': best_match.get('disc_number', 1)
        }

    print(f"⚠️ Could not confidently match track '{slsk_track_meta['title']}'. Using original metadata.")
    return slsk_track_meta # Fallback to original



# --- Post-Processing Logic ---
def _post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id):
    """
    NEW VERIFICATION WORKFLOW: Enhanced post-processing with file verification.
    Only sets task status to 'completed' after successful file verification and move operation.
    """
    _pp = pp_logger
    try:
        # Call the existing post-processing logic (but skip its completion callback)
        # We'll handle the completion callback ourselves after verification
        original_task_id = context.pop('task_id', None)  # Temporarily remove to prevent double callback
        original_batch_id = context.pop('batch_id', None)
        _post_process_matched_download(context_key, context, file_path)
        # Restore the IDs for our own callback
        if original_task_id:
            context['task_id'] = original_task_id
        if original_batch_id:
            context['batch_id'] = original_batch_id

        # Check if AcoustID quarantined the file — no further processing needed
        if context.get('_acoustid_quarantined'):
            failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed')
            _pp.info(f"File was quarantined by AcoustID verification (task={task_id}): {failure_msg}")
            with tasks_lock:
                if task_id in download_tasks:
                    download_tasks[task_id]['status'] = 'failed'
                    download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {failure_msg}"
            with matched_context_lock:
                if context_key in matched_downloads_context:
                    del matched_downloads_context[context_key]
            _on_download_completed(batch_id, task_id, success=False)
            return

        # Check if simple download handler already completed everything
        if context.get('_simple_download_completed'):
            expected_final_path = context.get('_final_path')

            if expected_final_path and os.path.exists(expected_final_path):
                with tasks_lock:
                    if task_id in download_tasks:
                        _mark_task_completed(task_id, context.get('track_info'))

                with matched_context_lock:
                    if context_key in matched_downloads_context:
                        del matched_downloads_context[context_key]

                _on_download_completed(batch_id, task_id, success=True)
                return
            else:
                _pp.info(f"FAILED simple download file not found at: {expected_final_path} (task={task_id}, context={context_key})")
                with tasks_lock:
                    if task_id in download_tasks:
                        download_tasks[task_id]['status'] = 'failed'
                        download_tasks[task_id]['error_message'] = f'Downloaded file not found at expected location: {os.path.basename(expected_final_path)}'

                with matched_context_lock:
                    if context_key in matched_downloads_context:
                        del matched_downloads_context[context_key]

                _on_download_completed(batch_id, task_id, success=False)
                return

        # VERIFICATION: Use the actual path that _post_process_matched_download computed and
        # moved the file to, instead of recomputing independently. Independent recomputation
        # can produce path mismatches (e.g., track_number extraction/validation differences)
        # causing false "file verification failed" errors on successfully processed files.
        expected_final_path = context.get('_final_processed_path')
        if not expected_final_path:
            _pp.info(f"⚠️ No _final_processed_path in context for task {task_id} — cannot verify, assuming success")
            with tasks_lock:
                if task_id in download_tasks:
                    _mark_task_completed(task_id, context.get('track_info'))
            with matched_context_lock:
                if context_key in matched_downloads_context:
                    del matched_downloads_context[context_key]
            _on_download_completed(batch_id, task_id, success=True)
            return

        # VERIFICATION: Check if file exists at the path processing actually used
        if os.path.exists(expected_final_path):
            # Mark task as completed only after successful verification
            with tasks_lock:
                if task_id in download_tasks:
                    _mark_task_completed(task_id, context.get('track_info'))
                    download_tasks[task_id]['metadata_enhanced'] = True

            with matched_context_lock:
                if context_key in matched_downloads_context:
                    del matched_downloads_context[context_key]

            _on_download_completed(batch_id, task_id, success=True)
        else:
            # Log failure details for diagnosis
            track_name = context.get('original_search_result', {}).get('spotify_clean_title', context_key)
            _pp.info(f"FAILED verification for '{track_name}' (task={task_id})")
            _pp.info(f"  expected_final_path: {expected_final_path}")
            _pp.info(f"  file_path (source): {file_path}, exists={os.path.exists(file_path)}")
            _pp.info(f"  is_album={context.get('is_album_download', False)}, has_clean_data={context.get('has_clean_spotify_data', False)}")
            expected_dir = os.path.dirname(expected_final_path)
            if os.path.exists(expected_dir):
                dir_contents = os.listdir(expected_dir)
                _pp.info(f"  directory contains {len(dir_contents)} files: {dir_contents[:20]}")
            else:
                _pp.info(f"  directory does not exist: {expected_dir}")

            with tasks_lock:
                if task_id in download_tasks:
                    download_tasks[task_id]['status'] = 'failed'
                    download_tasks[task_id]['error_message'] = f'File verification failed: expected file at {os.path.basename(expected_final_path)} but it was not found after processing'

            with matched_context_lock:
                if context_key in matched_downloads_context:
                    del matched_downloads_context[context_key]

            _on_download_completed(batch_id, task_id, success=False)

    except Exception as e:
        import traceback
        _pp.info(f"EXCEPTION in post-processing for '{context_key}' (task={task_id}): {e}")
        _pp.info(traceback.format_exc())
        with tasks_lock:
            if task_id in download_tasks:
                download_tasks[task_id]['status'] = 'failed'
                download_tasks[task_id]['error_message'] = f"Post-processing verification failed: {str(e)}"

        with matched_context_lock:
            if context_key in matched_downloads_context:
                del matched_downloads_context[context_key]

        _on_download_completed(batch_id, task_id, success=False)


def _check_flac_bit_depth(file_path, context, context_key):
    """
    Check if a FLAC file matches the user's preferred bit depth.
    Returns True if the file was rejected (caller should return), False if OK.
    """
    if not context.get('_audio_quality', '').startswith('FLAC'):
        return False

    from database.music_database import MusicDatabase
    _qp_db = MusicDatabase()
    _quality_profile = _qp_db.get_quality_profile()
    _flac_pref = _quality_profile.get('qualities', {}).get('flac', {}).get('bit_depth', 'any')

    if _flac_pref == 'any':
        return False

    # Parse actual bit depth from quality string like "FLAC 16bit"
    _actual_bits = context['_audio_quality'].replace('FLAC ', '').replace('bit', '')
    if _actual_bits == _flac_pref:
        return False

    # Reject — same pattern as AcoustID verification failure
    rejection_msg = f"FLAC bit depth mismatch: file is {_actual_bits}-bit, preference is {_flac_pref}-bit"
    try:
        quarantine_path = _move_to_quarantine(file_path, context, rejection_msg)
        print(f"🚫 File quarantined due to bit depth filter: {quarantine_path}")
    except Exception as quarantine_error:
        logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}")
        try:
            os.remove(file_path)
        except Exception:
            pass

    context['_bitdepth_rejected'] = True
    with matched_context_lock:
        if context_key in matched_downloads_context:
            del matched_downloads_context[context_key]

    task_id = context.get('task_id')
    batch_id = context.get('batch_id')
    if task_id:
        with tasks_lock:
            if task_id in download_tasks:
                download_tasks[task_id]['status'] = 'failed'
                download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_msg}"
    if task_id and batch_id:
        _on_download_completed(batch_id, task_id, success=False)
    return True


def _move_to_quarantine(file_path: str, context: dict, reason: str) -> str:
    """
    Move a file to quarantine folder when AcoustID verification fails.
    Creates a JSON sidecar file with metadata about why the file was quarantined.

    Args:
        file_path: Original file path
        context: Download context with track info
        reason: Reason for quarantine

    Returns:
        Path to quarantined file
    """
    import json
    from pathlib import Path
    from datetime import datetime

    # Get quarantine directory (inside download folder — always writable, even in Docker)
    download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
    quarantine_dir = Path(download_dir) / "ss_quarantine"
    quarantine_dir.mkdir(parents=True, exist_ok=True)

    # Create quarantine entry with timestamp
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    original_name = Path(file_path).stem
    file_ext = Path(file_path).suffix

    # Build quarantine filename: TIMESTAMP_originalname.ext.quarantined
    # The .quarantined extension prevents audio file searches and media servers
    # from picking up known-wrong files sitting in the downloads folder.
    quarantine_filename = f"{timestamp}_{original_name}{file_ext}.quarantined"
    quarantine_path = quarantine_dir / quarantine_filename

    # Move file to quarantine
    _safe_move_file(file_path, str(quarantine_path))

    # Write metadata sidecar file
    metadata_path = quarantine_dir / f"{timestamp}_{original_name}.json"

    # Extract track info from context
    track_info = context.get('track_info', {})
    original_search = context.get('original_search_result', {})
    spotify_artist = context.get('spotify_artist', {})

    metadata = {
        'original_filename': Path(file_path).name,
        'quarantine_reason': reason,
        'timestamp': datetime.now().isoformat(),
        'expected_track': (
            original_search.get('spotify_clean_title') or
            track_info.get('name') or
            original_search.get('title', 'Unknown')
        ),
        'expected_artist': spotify_artist.get('name', 'Unknown'),
        'context_key': context.get('context_key', 'unknown')
    }

    try:
        with open(metadata_path, 'w', encoding='utf-8') as f:
            json.dump(metadata, f, indent=2, ensure_ascii=False)
    except Exception as e:
        logger.warning(f"Failed to write quarantine metadata: {e}")

    logger.warning(f"🚫 File quarantined: {quarantine_path} - Reason: {reason}")

    try:
        if automation_engine:
            ti = context.get('track_info', {})
            artists = ti.get('artists', [])
            artist_name = ''
            if artists:
                a = artists[0]
                artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a)
            automation_engine.emit('download_quarantined', {
                'artist': artist_name,
                'title': ti.get('name', ''),
                'reason': reason or 'Unknown',
            })
    except Exception:
        pass

    return str(quarantine_path)


def _safe_move_file(src, dst):
    """
    Safely move a file across different filesystems/volumes.
    Handles Docker volume mount issues where shutil.move() fails on metadata preservation.

    Args:
        src: Source file path (str or Path)
        dst: Destination file path (str or Path)
    """
    import shutil
    from pathlib import Path

    src = Path(src)
    dst = Path(dst)

    # Ensure parent directory exists
    dst.parent.mkdir(parents=True, exist_ok=True)

    # If source doesn't exist, check if it was already moved to destination
    # This happens when a retry or parallel thread already transferred the file
    if not src.exists():
        if dst.exists():
            logger.info(f"Source gone but destination exists, file already transferred: {dst.name}")
            return
        else:
            raise FileNotFoundError(f"Source file not found and destination does not exist: {src}")

    # On Windows, shutil.move fails with FileExistsError if destination exists.
    # Remove it first, retrying briefly for Windows file locks (e.g. Plex scanning).
    if dst.exists():
        for _attempt in range(3):
            try:
                dst.unlink()
                break
            except PermissionError:
                if _attempt < 2:
                    time.sleep(1)
                else:
                    logger.warning(f"Could not remove locked destination after 3 attempts: {dst.name}")
            except Exception:
                break

    try:
        # Try standard move first (works if same filesystem)
        shutil.move(str(src), str(dst))
        return
    except FileNotFoundError:
        # Source vanished between our exists() check and the move - another thread got it first
        # If destination now exists, the other thread completed the transfer successfully
        if dst.exists():
            logger.info(f"Source moved by another thread, destination exists: {dst.name}")
            return
        raise
    except (OSError, PermissionError) as e:
        error_msg = str(e).lower()

        # shutil.move may have already copied the file successfully but failed
        # to delete the source (e.g. permission denied on slskd-owned downloads).
        # If destination exists with content, treat as success.
        if dst.exists() and dst.stat().st_size > 0:
            logger.warning(f"⚠️ Move raised {type(e).__name__} but destination exists, treating as success: {e}")
            # Try to clean up source, but don't fail if we can't
            try:
                src.unlink()
            except Exception:
                logger.info(f"Could not delete source file (may be owned by another process): {src}")
            return

        # Cross-device link error — do manual binary copy
        if "cross-device" in error_msg or "operation not permitted" in error_msg or "permission denied" in error_msg:
            logger.warning(f"⚠️ Cross-device move detected, using fallback copy method: {e}")
            try:
                # Simple copy without metadata preservation (avoids permission errors)
                with open(src, 'rb') as f_src:
                    with open(dst, 'wb') as f_dst:
                        shutil.copyfileobj(f_src, f_dst)
                        f_dst.flush()
                        os.fsync(f_dst.fileno())

                # Delete source after successful copy
                try:
                    src.unlink()
                except PermissionError:
                    logger.info(f"Could not delete source file (may be owned by another process): {src}")
                logger.info(f"✅ Successfully moved file using fallback method: {src} -> {dst}")
                return
            except Exception as fallback_error:
                logger.error(f"❌ Fallback copy also failed: {fallback_error}")
                raise
        else:
            # Re-raise if it's a different error
            raise


def _post_process_matched_download(context_key, context, file_path):
    """
    This is the final, corrected post-processing function. It now mirrors the
    GUI's logic by trusting the pre-matched context for album downloads, which
    solves the track numbering issue.

    Also handles simple downloads (from search page "Download" button) which
    just move files to /Transfer without metadata enhancement.
    """
    # --- PER-FILE LOCK ---
    # Acquire a per-context-key lock so only one thread processes a given file at a time.
    # The Stream Processor and Verification Worker both call this function for the same file.
    # Without serialization, they race to move the source file and the loser gets FileNotFoundError.
    # With the lock, the second thread waits, then the existing protection checks detect
    # "source gone + destination exists" and return early.
    with _post_process_locks_lock:
        if context_key not in _post_process_locks:
            _post_process_locks[context_key] = threading.Lock()
        file_lock = _post_process_locks[context_key]

    file_lock.acquire()
    try:
        import os
        import shutil
        import time
        from pathlib import Path

        # --- FILE STABILITY CHECK ---
        # Wait for the file to stop growing before processing. slskd may still be
        # flushing write buffers when it reports "Completed". We poll the file size
        # a few times; once it stabilises we know the write is finished.
        _basename = os.path.basename(file_path)
        _prev_size = -1
        for _stability_check in range(5):
            try:
                _cur_size = os.path.getsize(file_path)
            except OSError:
                _cur_size = -1
            if _cur_size == _prev_size and _cur_size > 0:
                # Size unchanged — file is stable
                break
            _prev_size = _cur_size
            if _stability_check == 0:
                print(f"⏳ Waiting for file to stabilise: {_basename} ({_cur_size} bytes)")
            time.sleep(1.5)
        else:
            print(f"⚠️ File may still be writing after stability checks: {_basename} ({_prev_size} bytes)")
        # --- END FILE STABILITY CHECK ---

        # --- ACOUSTID VERIFICATION ---
        # Optional verification that downloaded audio matches expected track.
        # Only runs if enabled and configured. Fails gracefully (skips on any error).
        try:
            from core.acoustid_verification import AcoustIDVerification, VerificationResult

            verifier = AcoustIDVerification()
            available, available_reason = verifier.quick_check_available()

            if available:
                # Extract expected track info from context
                track_info = context.get('track_info', {})
                original_search = context.get('original_search_result', {})
                spotify_artist = context.get('spotify_artist', {})

                expected_track = (
                    original_search.get('spotify_clean_title') or
                    track_info.get('name') or
                    original_search.get('title', '')
                )

                # Use track-level artist for verification, NOT album artist.
                # For compilations, spotify_artist is "Various Artists" which
                # will never match AcoustID's actual track artist.
                expected_artist = ''
                track_artists = track_info.get('artists', [])
                if track_artists:
                    first = track_artists[0]
                    if isinstance(first, dict):
                        expected_artist = first.get('name', '')
                    elif isinstance(first, str):
                        expected_artist = first
                # Fallback to album artist if no track artists available
                if not expected_artist:
                    expected_artist = spotify_artist.get('name', '')

                if expected_track and expected_artist:
                    print(f"🔍 Running AcoustID verification for: '{expected_track}' by '{expected_artist}'")
                    verification_result, verification_msg = verifier.verify_audio_file(
                        file_path,
                        expected_track,
                        expected_artist,
                        context
                    )
                    print(f"🔍 AcoustID verification result: {verification_result.value} - {verification_msg}")

                    if verification_result == VerificationResult.FAIL:
                        # Move to quarantine instead of Transfer
                        try:
                            quarantine_path = _move_to_quarantine(file_path, context, verification_msg)
                            print(f"🚫 File quarantined due to verification failure: {quarantine_path}")
                        except Exception as quarantine_error:
                            # Quarantine failed — delete the known-wrong file instead
                            # NEVER save a file we've confirmed is wrong
                            logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}")
                            print(f"🚫 Quarantine failed, deleting wrong file: {file_path}")
                            try:
                                os.remove(file_path)
                            except Exception as del_error:
                                logger.error(f"Could not delete wrong file either: {del_error}")

                        # These always execute for FAIL — whether quarantine succeeded or not
                        context['_acoustid_quarantined'] = True
                        context['_acoustid_failure_msg'] = verification_msg

                        # Clean up context
                        with matched_context_lock:
                            if context_key in matched_downloads_context:
                                del matched_downloads_context[context_key]

                        # Mark as failed in download tasks if we have task info
                        task_id = context.get('task_id')
                        batch_id = context.get('batch_id')
                        if task_id:
                            with tasks_lock:
                                if task_id in download_tasks:
                                    download_tasks[task_id]['status'] = 'failed'
                                    download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {verification_msg}"

                        # Call completion callback with failure
                        if task_id and batch_id:
                            _on_download_completed(batch_id, task_id, success=False)

                        return  # NEVER continue processing a known-wrong file
                else:
                    print(f"⚠️ AcoustID verification skipped: missing track/artist info")
            else:
                print(f"ℹ️ AcoustID verification not available: {available_reason}")
        except Exception as verify_error:
            # Any verification error should NOT block the download - fail open
            print(f"⚠️ AcoustID verification error (continuing normally): {verify_error}")
        # --- END ACOUSTID VERIFICATION ---

        # --- SIMPLE DOWNLOAD HANDLING ---
        # Check if this is a simple download (search page "Download ⬇" button only)
        search_result = context.get('search_result', {})
        is_simple_download = search_result.get('is_simple_download', False)

        if is_simple_download:
            # Simple transfer: move to Transfer folder, no metadata enhancement
            logger.info(f"Processing simple download (no metadata enhancement): {file_path}")

            transfer_path = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer'))

            # Check if this download has album info (from path or search result)
            album_name = None
            original_filename = search_result.get('filename', '')

            if '/' in original_filename or '\\' in original_filename:
                # Get parent directory as album name
                path_parts = original_filename.replace('\\', '/').split('/')
                if len(path_parts) >= 2:
                    album_name = path_parts[-2]  # Parent directory

            # If no album from path, check search result
            if not album_name:
                album_name = search_result.get('album')

            # Determine destination
            filename = Path(file_path).name

            if album_name and album_name.lower() not in ['unknown', 'unknown album', '']:
                # Has album info - create album folder
                import re
                album_name = re.sub(r'[<>:"/\\|?*]', '_', album_name).strip()
                album_folder = Path(transfer_path) / album_name
                album_folder.mkdir(parents=True, exist_ok=True)
                destination = album_folder / filename
                logger.info(f"Moving to album folder: {album_name}")
            else:
                # No album info - move directly to Transfer root (singles)
                Path(transfer_path).mkdir(parents=True, exist_ok=True)
                destination = Path(transfer_path) / filename
                logger.info(f"Moving to Transfer root (single track)")

            _safe_move_file(file_path, destination)
            logger.info(f"✅ Moved simple download to: {destination}")

            # Clean up context
            with matched_context_lock:
                if context_key in matched_downloads_context:
                    del matched_downloads_context[context_key]

            # Trigger library scan (using correct method name)
            if web_scan_manager:
                threading.Thread(
                    target=lambda: web_scan_manager.request_scan("Simple download completed"),
                    daemon=True
                ).start()

            add_activity_item("✅", "Download Complete", f"{album_name}/{filename}", "Now")
            logger.info(f"✅ Simple download post-processing complete: {album_name}/{filename}")

            # Set flag in context so verification function knows this was fully handled
            context['_simple_download_completed'] = True
            context['_final_path'] = str(destination)
            _emit_track_downloaded(context)
            return
        # --- END SIMPLE DOWNLOAD HANDLING ---

        print(f"🎯 Starting robust post-processing for: {context_key}")

        spotify_artist = context.get("spotify_artist")
        if not spotify_artist:
            print(f"❌ Post-processing failed: Missing spotify_artist context.")
            return

        # Check if playlist folder mode is enabled (sync page playlists only)
        track_info = context.get("track_info", {})
        playlist_folder_mode = track_info.get("_playlist_folder_mode", False)

        print(f"🔍 [Debug] Post-processing - track_info type: {type(track_info)}, is None: {track_info is None}, is empty: {not track_info}")
        print(f"🔍 [Debug] Post-processing - playlist_folder_mode: {playlist_folder_mode}")
        if track_info:
            print(f"🔍 [Debug] Post-processing - track_info keys: {list(track_info.keys())}")

        if playlist_folder_mode:
            # Use shared path builder for playlist mode
            playlist_name = track_info.get("_playlist_name", "Unknown Playlist")
            print(f"📁 [Playlist Folder Mode] Organizing in playlist folder: {playlist_name}")

            file_ext = os.path.splitext(file_path)[1]
            context['_audio_quality'] = _get_audio_quality_string(file_path)
            if context['_audio_quality']:
                print(f"🎧 Audio quality detected: {context['_audio_quality']}")

            # FLAC bit depth filter
            if _check_flac_bit_depth(file_path, context, context_key):
                return

            final_path, _ = _build_final_path_for_track(context, spotify_artist, None, file_ext)
            print(f"📁 Playlist mode final path: '{final_path}'")

            # Enhance metadata before moving
            try:
                print(f"🔍 [Metadata Input] Playlist mode - artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})")
                _enhance_file_metadata(file_path, context, spotify_artist, None)
            except Exception as meta_err:
                import traceback
                pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")

            # Move file to playlist folder
            print(f"🚚 Moving '{os.path.basename(file_path)}' to '{final_path}'")
            if os.path.exists(final_path):
                print(f"⚠️ File already exists, overwriting: {os.path.basename(final_path)}")
                os.remove(final_path)

            _safe_move_file(file_path, final_path)

            # Lossy copy: create MP3 version if enabled
            blasphemy_path = _create_lossy_copy(final_path)
            if blasphemy_path:
                context['_final_processed_path'] = blasphemy_path

            # Clean up empty directories in downloads folder
            downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
            _cleanup_empty_directories(downloads_path, file_path)

            print(f"✅ [Playlist Folder Mode] Post-processing complete: {context.get('_final_processed_path', final_path)}")

            # WISHLIST REMOVAL: Check if this track should be removed from wishlist
            try:
                _check_and_remove_from_wishlist(context)
            except Exception as wishlist_error:
                print(f"⚠️ [Playlist Folder] Error checking wishlist removal: {wishlist_error}")

            _emit_track_downloaded(context)

            # NOTE: Don't call callbacks here - let verification function handle completion
            # The verification function will check file exists and then call callbacks
            return  # Skip normal album/artist folder structure processing

        is_album_download = context.get("is_album_download", False)
        has_clean_spotify_data = context.get("has_clean_spotify_data", False)

        if is_album_download and has_clean_spotify_data:
            # Build album_info directly from clean Spotify metadata (GUI PARITY)
            print("✅ Album context with clean Spotify data found - using direct album info")
            original_search = context.get("original_search_result", {})
            spotify_album = context.get("spotify_album", {})
            
            # Use clean Spotify metadata (matches GUI's SpotifyBasedSearchResult approach)
            clean_track_name = original_search.get('spotify_clean_title', 'Unknown Track')
            clean_album_name = original_search.get('spotify_clean_album', 'Unknown Album')
            
            # DEBUG: Check what's in original_search
            print(f"🔍 [DEBUG] Path 1 - Clean Spotify data path:")
            print(f"   original_search keys: {list(original_search.keys())}")
            print(f"   track_number in original_search: {'track_number' in original_search}")
            print(f"   track_number value: {original_search.get('track_number', 'NOT_FOUND')}")
            
            album_info = {
                'is_album': True,
                'album_name': clean_album_name,  # Use clean Spotify album name
                'track_number': original_search.get('track_number', 1),
                'disc_number': original_search.get('disc_number', 1),
                'clean_track_name': clean_track_name,
                'album_image_url': spotify_album.get('image_url'),
                'confidence': 1.0,  # High confidence since we have clean Spotify data
                'source': 'clean_spotify_metadata'
            }

            print(f"🎯 Using clean Spotify album: '{clean_album_name}' for track: '{clean_track_name}'")
        elif is_album_download:
            # CRITICAL FIX: Album context without clean Spotify data - still force album treatment
            print("⚠️ Album context found but no clean Spotify data - using enhanced fallback")
            original_search = context.get("original_search_result", {})
            spotify_album = context.get("spotify_album", {})
            clean_track_name = original_search.get('spotify_clean_title') or original_search.get('title', 'Unknown Track')
            
            # DEBUG: Check what's in original_search for path 2
            print(f"🔍 [DEBUG] Path 2 - Enhanced fallback album context path:")
            print(f"   original_search keys: {list(original_search.keys())}")
            print(f"   track_number in original_search: {'track_number' in original_search}")
            print(f"   track_number value: {original_search.get('track_number', 'NOT_FOUND')}")
            print(f"   spotify_album name: {spotify_album.get('name', 'NOT_FOUND')}")
            
            # ENHANCEMENT: Use spotify_clean_album if available for consistency 
            album_name = (original_search.get('spotify_clean_album') or 
                         spotify_album.get('name') or 
                         'Unknown Album')
            
            album_info = {
                'is_album': True,  # FORCE TRUE - user explicitly selected album for download
                'album_name': album_name,
                'track_number': original_search.get('track_number', 1),
                'disc_number': original_search.get('disc_number', 1),
                'clean_track_name': clean_track_name,
                'album_image_url': spotify_album.get('image_url'),
                'confidence': 0.9,  # Higher confidence - user explicitly chose album
                'source': 'enhanced_fallback_album_context'
            }
            print(f"🎯 [FORCED ALBUM] Using album: '{album_name}' for track: '{clean_track_name}'")
        else:
            # For singles, we still need to detect if they belong to an album.
            print("🎵 Single track download - attempting album detection")
            album_info = _detect_album_info_web(context, spotify_artist)

        # --- Album grouping resolution ---
        # Only run smart grouping for singles/auto-detected albums.
        # Explicit album downloads already have the correct Spotify album name —
        # re-grouping would mangle names like "(Reworked and Remastered)" into "(Deluxe Edition)".
        if album_info and album_info['is_album'] and not is_album_download:
            print(f"\n🎯 SMART ALBUM GROUPING for track: '{album_info.get('clean_track_name', 'Unknown')}'")
            print(f"   Original album: '{album_info.get('album_name', 'None')}'")

            # Get original album name from context if available
            original_album = None
            if context.get("original_search_result", {}).get("album"):
                original_album = context["original_search_result"]["album"]

            # Use the GUI's smart album grouping algorithm
            consistent_album_name = _resolve_album_group(spotify_artist, album_info, original_album)
            album_info['album_name'] = consistent_album_name

            print(f"   Final album name: '{consistent_album_name}'")
            print(f"🔗 ✅ Album grouping complete!\n")
        elif album_info and album_info['is_album'] and is_album_download:
            print(f"\n🎯 EXPLICIT ALBUM DOWNLOAD - preserving Spotify album name: '{album_info.get('album_name', 'None')}'")
            print(f"   Skipping smart grouping (not needed for explicit album downloads)\n")

        # 1. Get transfer path and create artist directory
        transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer'))
        artist_name_sanitized = _sanitize_filename(spotify_artist["name"])
        artist_dir = os.path.join(transfer_dir, artist_name_sanitized)
        os.makedirs(artist_dir, exist_ok=True)
        
        file_ext = os.path.splitext(file_path)[1]
        context['_audio_quality'] = _get_audio_quality_string(file_path)
        if context['_audio_quality']:
            print(f"🎧 Audio quality detected: {context['_audio_quality']}")

        # FLAC bit depth filter
        if _check_flac_bit_depth(file_path, context, context_key):
            return

        # 2. Build the final path using GUI-style track naming with multiple fallback sources
        if album_info and album_info['is_album']:
            album_name_sanitized = _sanitize_filename(album_info['album_name'])
            
            # --- GUI PARITY: Use multiple sources for clean track name ---
            original_search = context.get("original_search_result", {})
            clean_track_name = album_info['clean_track_name']
            
            # Priority 1: Spotify clean title from context
            if original_search.get('spotify_clean_title'):
                clean_track_name = original_search['spotify_clean_title']
                print(f"🎵 Using Spotify clean title: '{clean_track_name}'")
            # Priority 2: Album info clean name
            elif album_info.get('clean_track_name'):
                clean_track_name = album_info['clean_track_name']
                print(f"🎵 Using album info clean name: '{clean_track_name}'")
            # Priority 3: Original title as fallback
            else:
                clean_track_name = original_search.get('title', 'Unknown Track')
                print(f"🎵 Using original title as fallback: '{clean_track_name}'")
            
            final_track_name_sanitized = _sanitize_filename(clean_track_name)
            track_number = album_info['track_number']
            
            # DEBUG: Check final track_number values
            print(f"🔍 [DEBUG] Final track_number processing:")
            print(f"   album_info source: {album_info.get('source', 'unknown')}")
            print(f"   album_info track_number: {album_info.get('track_number', 'NOT_FOUND')}")
            print(f"   track_number variable: {track_number}")
            
            # Fix: Handle None track_number
            if track_number is None:
                print(f"⚠️ Track number is None, extracting from filename: {os.path.basename(file_path)}")
                track_number = _extract_track_number_from_filename(file_path)
                print(f"   -> Extracted track number: {track_number}")
            
            # Ensure track_number is valid
            if not isinstance(track_number, int) or track_number < 1:
                print(f"⚠️ Invalid track number ({track_number}), defaulting to 1")
                track_number = 1
                
            print(f"🎯 [DEBUG] FINAL track_number used for filename: {track_number}")
            
            # CRITICAL FIX: Update album_info with corrected track_number for metadata enhancement
            album_info['track_number'] = track_number
            album_info['clean_track_name'] = clean_track_name  # Ensure clean name is in album_info
            print(f"✅ [FIX] Updated album_info track_number to {track_number} for consistent metadata")

            # Use shared path builder for album mode
            final_path, _ = _build_final_path_for_track(context, spotify_artist, album_info, file_ext)
            print(f"📁 Album path: '{final_path}'")
        else:
            # Single track structure: Transfer/ARTIST/ARTIST - SINGLE/SINGLE.ext
            # --- GUI PARITY: Use multiple sources for clean track name ---
            original_search = context.get("original_search_result", {})
            clean_track_name = album_info['clean_track_name']
            
            # Priority 1: Spotify clean title from context  
            if original_search.get('spotify_clean_title'):
                clean_track_name = original_search['spotify_clean_title']
                print(f"🎵 Using Spotify clean title: '{clean_track_name}'")
            # Priority 2: Album info clean name
            elif album_info and album_info.get('clean_track_name'):
                clean_track_name = album_info['clean_track_name']
                print(f"🎵 Using album info clean name: '{clean_track_name}'")
            # Priority 3: Original title as fallback
            else:
                clean_track_name = original_search.get('title', 'Unknown Track')
                print(f"🎵 Using original title as fallback: '{clean_track_name}'")
            
            # Ensure clean name is in album_info for path builder
            if album_info:
                album_info['clean_track_name'] = clean_track_name

            # Use shared path builder for single mode
            final_path, _ = _build_final_path_for_track(context, spotify_artist, album_info, file_ext)
            print(f"📁 Single path: '{final_path}'")

        # Store the actual computed path so verification uses this exact path
        # instead of recomputing independently (which can produce mismatches)
        context['_final_processed_path'] = final_path

        # 3. Enhance metadata, move file, download art, and cleanup
        try:
            print(f"🔍 [Metadata Input] artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})")
            if album_info:
                print(f"🔍 [Metadata Input] album: '{album_info.get('album_name', 'MISSING')}', track#: {album_info.get('track_number', 'MISSING')}, disc#: {album_info.get('disc_number', 'MISSING')}, source: {album_info.get('source', 'unknown')}")
            else:
                print(f"🔍 [Metadata Input] album_info: None (single track)")
            _enhance_file_metadata(file_path, context, spotify_artist, album_info)
        except Exception as meta_err:
            import traceback
            pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")
            # Continue anyway - file can still be moved

        print(f"🚚 Moving '{os.path.basename(file_path)}' to '{final_path}'")
        if os.path.exists(final_path):
            # PROTECTION: If destination already exists, check before overwriting
            # If the source file is gone, another thread already handled this - don't delete the destination
            if not os.path.exists(file_path):
                print(f"✅ [Protection] Destination exists and source already gone - file already transferred: {os.path.basename(final_path)}")
                return
            try:
                from mutagen import File as MutagenFile
                existing_file = MutagenFile(final_path)
                has_metadata = existing_file is not None and len(existing_file.tags or {}) > 2  # More than basic tags

                if has_metadata:
                    print(f"⚠️ [Protection] Existing file already has metadata enhancement - skipping overwrite: {os.path.basename(final_path)}")
                    print(f"🗑️ [Protection] Removing redundant download file: {os.path.basename(file_path)}")
                    try:
                        os.remove(file_path)  # Remove the redundant file
                    except FileNotFoundError:
                        print(f"⚠️ [Protection] Could not remove redundant file (already gone): {file_path}")
                    except Exception as e:
                        print(f"⚠️ [Protection] Error removing redundant file: {e}")
                    return  # Don't overwrite the good file
                else:
                    print(f"🔄 [Protection] Existing file lacks metadata - safe to overwrite: {os.path.basename(final_path)}")
                    try:
                        os.remove(final_path)
                    except FileNotFoundError:
                        pass # It was just there, but now gone?
            except Exception as check_error:
                print(f"⚠️ [Protection] Error checking existing file metadata, proceeding with overwrite: {check_error}")
                try:
                    if os.path.exists(final_path):
                        os.remove(final_path)
                except Exception as e:
                    print(f"⚠️ [Protection] Failed to remove existing file for overwrite: {e}")

        # --- PRE-MOVE SOURCE CHECK ---
        # Right before moving, verify the source file still exists.
        # Another thread (Stream Processor or Verification Worker) may have
        # already moved this file during the sleep + metadata enhancement window.
        if not os.path.exists(file_path):
            if os.path.exists(final_path):
                print(f"✅ [Pre-Move] Source already gone and destination exists - another thread completed transfer: {os.path.basename(final_path)}")
                # Still do cover art + lyrics since the other thread might not have finished those
                _download_cover_art(album_info, os.path.dirname(final_path))
                _generate_lrc_file(final_path, context, spotify_artist, album_info)
                return
            else:
                # Source gone, exact destination not found — check if stream processor already
                # moved it with a quality tag (e.g. "track [FLAC 24bit].flac" vs "track.flac")
                expected_dir = os.path.dirname(final_path)
                expected_stem = os.path.splitext(os.path.basename(final_path))[0]
                expected_ext = os.path.splitext(final_path)[1]
                found_variant = None
                # Also check for .mp3 if Blasphemy Mode may have deleted the .flac
                check_exts = {expected_ext}
                if expected_ext == '.flac' and config_manager.get('lossy_copy.enabled', False) and config_manager.get('lossy_copy.delete_original', False):
                    check_exts.add('.mp3')
                if os.path.exists(expected_dir):
                    for f in os.listdir(expected_dir):
                        # Match files that start with the expected stem and have a matching extension
                        # This catches "01 - track [FLAC 24bit].flac" when expecting "01 - track.flac"
                        # and "01 - track [MP3-320].mp3" when Blasphemy Mode deleted the FLAC
                        f_ext = os.path.splitext(f)[1].lower()
                        if f_ext in check_exts and os.path.splitext(f)[0].startswith(expected_stem):
                            found_variant = os.path.join(expected_dir, f)
                            break
                if found_variant:
                    print(f"✅ [Pre-Move] Source gone but found variant in destination (stream processor handled it): {os.path.basename(found_variant)}")
                    context['_final_processed_path'] = found_variant
                    _download_cover_art(album_info, expected_dir)
                    _generate_lrc_file(found_variant, context, spotify_artist, album_info)
                    return
                else:
                    print(f"⚠️ [Pre-Move] Source file gone and no matching file in destination: {os.path.basename(file_path)}")
                    raise FileNotFoundError(f"Source file vanished before move and destination does not exist: {file_path}")

        _safe_move_file(file_path, final_path)

        _download_cover_art(album_info, os.path.dirname(final_path))

        # 4. Generate LRC lyrics file at final location (elegant addition)
        _generate_lrc_file(final_path, context, spotify_artist, album_info)

        # Lossy copy: create MP3 version if enabled
        blasphemy_path = _create_lossy_copy(final_path)
        if blasphemy_path:
            context['_final_processed_path'] = blasphemy_path

        downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
        _cleanup_empty_directories(downloads_path, file_path)

        print(f"✅ Post-processing complete for: {context.get('_final_processed_path', final_path)}")

        _emit_track_downloaded(context)

        # RETAG DATA CAPTURE: Record completed album/single downloads for retag tool
        try:
            if not playlist_folder_mode:
                completed_path = context.get('_final_processed_path', final_path)
                _record_retag_download(context, spotify_artist, album_info, completed_path)
        except Exception as retag_err:
            print(f"⚠️ [Post-Process] Retag data capture failed (non-fatal): {retag_err}")

        # REPAIR: Register album folder for repair scanning when batch completes
        try:
            completed_path = context.get('_final_processed_path', final_path)
            batch_id_for_repair = context.get('batch_id')
            if completed_path and batch_id_for_repair and repair_worker:
                album_folder = os.path.dirname(str(completed_path))
                if album_folder:
                    repair_worker.register_folder(batch_id_for_repair, album_folder)
        except Exception as repair_err:
            print(f"⚠️ [Post-Process] Repair folder registration failed: {repair_err}")

        # WISHLIST REMOVAL: Check if this track should be removed from wishlist after successful download
        try:
            _check_and_remove_from_wishlist(context)
        except Exception as wishlist_error:
            print(f"⚠️ [Post-Process] Error checking wishlist removal: {wishlist_error}")
        
        # Call completion callback for missing downloads tasks to start next batch
        task_id = context.get('task_id')
        batch_id = context.get('batch_id')
        if task_id and batch_id:
            print(f"🎯 [Post-Process] Calling completion callback for task {task_id} in batch {batch_id}")
            
            # Mark task as stream processed and set terminal status so
            # _validate_worker_counts won't count this task as active
            # (prevents active_count inflation race).
            # NOTE: Only set status here — don't call _mark_task_completed() because
            # _run_post_processing_worker will call it later with the session counter
            # increment. Calling it here too would double-count the download.
            with tasks_lock:
                if task_id in download_tasks:
                    download_tasks[task_id]['stream_processed'] = True
                    download_tasks[task_id]['status'] = 'completed'
                    print(f"✅ [Post-Process] Marked task {task_id} as completed")
            
            _on_download_completed(batch_id, task_id, success=True)

    except Exception as e:
        import traceback
        pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: {e}")
        pp_logger.info(traceback.format_exc())
        print(f"\n❌ CRITICAL ERROR in post-processing for {context_key}: {e}")
        traceback.print_exc()

        # Only retry if the source file still exists - otherwise retrying is pointless
        # and creates an infinite loop of failures
        import os as _os
        source_exists = _os.path.exists(file_path) if file_path else False
        if source_exists:
            # Remove from processed set so it can be retried
            if context_key in _processed_download_ids:
                _processed_download_ids.remove(context_key)
                print(f"🔄 Removed {context_key} from processed set - will retry on next check")

            # Re-add to matched context for retry
            with matched_context_lock:
                if context_key not in matched_downloads_context:
                    matched_downloads_context[context_key] = context
                    print(f"♻️ Re-added {context_key} to context for retry")
        else:
            print(f"⚠️ Source file gone, not retrying: {context_key}")
    finally:
        file_lock.release()
        # Clean up the lock entry to prevent unbounded memory growth
        with _post_process_locks_lock:
            _post_process_locks.pop(context_key, None)

# Keep track of processed downloads to avoid re-processing
_processed_download_ids = set()

# Track stale transfer keys (completed in slskd but no context — e.g., from before app restart)
# so we only log the warning once per key instead of spamming every poll cycle
_stale_transfer_keys = set()

# Per-context-key locks to prevent two threads from processing the same file simultaneously.
# Without this, the Stream Processor and Verification Worker race to move the same source file,
# and the loser gets a FileNotFoundError because the winner already moved it.
_post_process_locks = {}  # {context_key: threading.Lock()}
_post_process_locks_lock = threading.Lock()  # protects the dict itself

# --- File Discovery Retry System ---
# Prevents race condition where slskd reports completion before file is written to disk
# Tracks retry attempts per download to give files time to finish writing
_download_retry_attempts = {}  # {context_key: {'count': N, 'first_attempt': timestamp}}
_download_retry_max = 10  # Max retries before giving up (10 seconds with 1s poll interval)
_download_retry_lock = threading.Lock()

def _record_retag_download(context, spotify_artist, album_info, final_path):
    """Record a completed download in the retag tables for later re-tagging."""
    from database.music_database import get_database
    db = get_database()

    # Extract artist name
    if isinstance(spotify_artist, dict):
        artist_name = spotify_artist.get('name', 'Unknown Artist')
    else:
        artist_name = getattr(spotify_artist, 'name', 'Unknown Artist')

    spotify_album = context.get('spotify_album', {})
    original_search = context.get('original_search_result', {})
    track_info = context.get('track_info', {})

    is_album = album_info and album_info.get('is_album', False)
    group_type = 'album' if is_album else 'single'
    album_name = album_info.get('album_name', '') if album_info else (
        original_search.get('spotify_clean_title', 'Unknown'))

    # Determine album IDs (Spotify vs iTunes)
    spotify_album_id = None
    itunes_album_id = None
    if spotify_album:
        album_id_raw = str(spotify_album.get('id', ''))
        if album_id_raw and album_id_raw.isdigit():
            itunes_album_id = album_id_raw
        elif album_id_raw:
            spotify_album_id = album_id_raw

    image_url = album_info.get('album_image_url') if album_info else None
    total_tracks = spotify_album.get('total_tracks', 1) if spotify_album else 1
    release_date = spotify_album.get('release_date', '') if spotify_album else ''

    # Find or create group (avoid duplicating for multi-track albums)
    group_id = db.find_retag_group(artist_name, album_name)
    if group_id is None:
        group_id = db.add_retag_group(
            group_type=group_type, artist_name=artist_name, album_name=album_name,
            image_url=image_url, spotify_album_id=spotify_album_id,
            itunes_album_id=itunes_album_id, total_tracks=total_tracks,
            release_date=release_date
        )
    if group_id is None:
        return

    # Track details
    track_number = album_info.get('track_number', 1) if album_info else 1
    disc_number = original_search.get('disc_number') or (
        album_info.get('disc_number', 1) if album_info else 1)
    title = original_search.get('spotify_clean_title') or (
        album_info.get('clean_track_name', 'Unknown Track') if album_info else 'Unknown Track')
    file_format = os.path.splitext(str(final_path))[1].lstrip('.').lower()

    # Track IDs (Spotify vs iTunes)
    spotify_track_id = None
    itunes_track_id = None
    if track_info and track_info.get('id'):
        tid = str(track_info['id'])
        if tid.isdigit():
            itunes_track_id = tid
        else:
            spotify_track_id = tid

    # Avoid duplicate track entries
    if not db.retag_track_exists(group_id, str(final_path)):
        db.add_retag_track(
            group_id=group_id, track_number=track_number, disc_number=disc_number,
            title=title, file_path=str(final_path), file_format=file_format,
            spotify_track_id=spotify_track_id, itunes_track_id=itunes_track_id
        )
        print(f"📝 [Retag] Recorded track for retag: '{title}' in '{album_name}'")

    # Cap retag groups at 100, remove oldest
    db.trim_retag_groups(100)


def _execute_retag(group_id, album_id):
    """Execute a retag operation: re-tag files in a group with metadata from a new album match."""
    global retag_state
    from database.music_database import get_database

    try:
        with retag_lock:
            retag_state.update({
                "status": "running",
                "phase": "Fetching album metadata...",
                "progress": 0,
                "current_track": "",
                "total_tracks": 0,
                "processed": 0,
                "error_message": ""
            })

        # 1. Fetch new album metadata from Spotify/iTunes
        album_data = spotify_client.get_album(album_id)
        if not album_data:
            raise ValueError(f"Could not fetch album data for ID: {album_id}")

        album_tracks_response = spotify_client.get_album_tracks(album_id)
        if not album_tracks_response:
            raise ValueError(f"Could not fetch album tracks for ID: {album_id}")

        album_tracks_items = album_tracks_response.get('items', [])

        # Extract artist info
        album_artists = album_data.get('artists', [])
        new_artist = album_artists[0] if album_artists else {'name': 'Unknown Artist', 'id': ''}
        # Ensure artist is a dict with expected fields
        if not isinstance(new_artist, dict):
            new_artist = {'name': str(new_artist), 'id': ''}
        new_album_name = album_data.get('name', 'Unknown Album')
        new_images = album_data.get('images', [])
        new_image_url = new_images[0]['url'] if new_images else None
        new_release_date = album_data.get('release_date', '')
        total_tracks = album_data.get('total_tracks', len(album_tracks_items))

        # Build spotify track list
        spotify_tracks = []
        for item in album_tracks_items:
            track_artists = item.get('artists', [])
            spotify_tracks.append({
                'name': item.get('name', ''),
                'track_number': item.get('track_number', 1),
                'disc_number': item.get('disc_number', 1),
                'id': item.get('id', ''),
                'artists': track_artists,
                'duration_ms': item.get('duration_ms', 0)
            })

        total_discs = max((t['disc_number'] for t in spotify_tracks), default=1)

        # 2. Load existing tracks for this group
        db = get_database()
        existing_tracks = db.get_retag_tracks(group_id)
        if not existing_tracks:
            raise ValueError(f"No tracks found for retag group {group_id}")

        with retag_lock:
            retag_state['total_tracks'] = len(existing_tracks)
            retag_state['phase'] = "Matching tracks..."

        # 3. Match existing files to new tracklist
        matched_pairs = []
        for existing_track in existing_tracks:
            best_match = None
            best_score = 0

            # Priority 1: Match by track number
            for st in spotify_tracks:
                if (st['track_number'] == existing_track.get('track_number') and
                        st['disc_number'] == existing_track.get('disc_number', 1)):
                    best_match = st
                    best_score = 1.0
                    break

            # Priority 2: Match by title similarity
            if not best_match:
                from difflib import SequenceMatcher
                existing_title = (existing_track.get('title') or '').lower().strip()
                for st in spotify_tracks:
                    st_title = (st.get('name') or '').lower().strip()
                    score = SequenceMatcher(None, existing_title, st_title).ratio()
                    if score > best_score and score > 0.6:
                        best_score = score
                        best_match = st

            if best_match:
                matched_pairs.append((existing_track, best_match))
            else:
                print(f"⚠️ [Retag] No match found for track: '{existing_track.get('title')}'")
                matched_pairs.append((existing_track, None))

        with retag_lock:
            retag_state['phase'] = "Retagging files..."

        # 4. Retag each matched track
        for existing_track, matched_spotify in matched_pairs:
            current_file_path = existing_track.get('file_path', '')
            track_title = matched_spotify['name'] if matched_spotify else existing_track.get('title', 'Unknown')

            with retag_lock:
                retag_state['current_track'] = track_title

            if not matched_spotify:
                with retag_lock:
                    retag_state['processed'] += 1
                    retag_state['progress'] = int(retag_state['processed'] / retag_state['total_tracks'] * 100)
                continue

            # Verify file exists
            if not os.path.exists(current_file_path):
                print(f"⚠️ [Retag] File not found, skipping: {current_file_path}")
                with retag_lock:
                    retag_state['processed'] += 1
                    retag_state['progress'] = int(retag_state['processed'] / retag_state['total_tracks'] * 100)
                continue

            # Build synthetic context for _enhance_file_metadata
            track_artists = matched_spotify.get('artists', [])
            context = {
                'original_search_result': {
                    'spotify_clean_title': matched_spotify['name'],
                    'spotify_clean_album': new_album_name,
                    'track_number': matched_spotify['track_number'],
                    'disc_number': matched_spotify.get('disc_number', 1),
                    'artists': track_artists,
                    'title': matched_spotify['name']
                },
                'spotify_album': {
                    'id': album_id,
                    'name': new_album_name,
                    'release_date': new_release_date,
                    'total_tracks': total_tracks,
                    'image_url': new_image_url,
                    'total_discs': total_discs
                },
                'track_info': {'id': matched_spotify['id']},
                'spotify_artist': new_artist,
                '_audio_quality': _get_audio_quality_string(current_file_path) or ''
            }

            album_info = {
                'is_album': total_tracks > 1,
                'album_name': new_album_name,
                'track_number': matched_spotify['track_number'],
                'disc_number': matched_spotify.get('disc_number', 1),
                'clean_track_name': matched_spotify['name'],
                'album_image_url': new_image_url
            }

            # Re-write metadata tags
            try:
                _enhance_file_metadata(current_file_path, context, new_artist, album_info)
                print(f"✅ [Retag] Re-tagged: '{track_title}'")
            except Exception as meta_err:
                print(f"⚠️ [Retag] Metadata write failed for '{track_title}': {meta_err}")

            # Compute new path and move if different
            file_ext = os.path.splitext(current_file_path)[1]
            try:
                new_path, _ = _build_final_path_for_track(context, new_artist, album_info, file_ext)

                if os.path.normpath(current_file_path) != os.path.normpath(new_path):
                    print(f"🚚 [Retag] Moving '{os.path.basename(current_file_path)}' -> '{new_path}'")
                    old_dir = os.path.dirname(current_file_path)
                    os.makedirs(os.path.dirname(new_path), exist_ok=True)
                    _safe_move_file(current_file_path, new_path)

                    # Move .lrc file alongside audio file if it exists
                    old_lrc = os.path.splitext(current_file_path)[0] + '.lrc'
                    if os.path.exists(old_lrc):
                        new_lrc = os.path.splitext(new_path)[0] + '.lrc'
                        try:
                            _safe_move_file(old_lrc, new_lrc)
                            print(f"📝 [Retag] Moved .lrc file alongside audio")
                        except Exception as lrc_err:
                            print(f"⚠️ [Retag] Failed to move .lrc file: {lrc_err}")

                    # Remove old cover.jpg if directory changed and old dir is now empty of audio
                    new_dir = os.path.dirname(new_path)
                    if os.path.normpath(old_dir) != os.path.normpath(new_dir):
                        old_cover = os.path.join(old_dir, 'cover.jpg')
                        if os.path.exists(old_cover):
                            # Check if any audio files remain in old directory
                            audio_exts = {'.flac', '.mp3', '.m4a', '.ogg', '.opus', '.wav', '.aac'}
                            remaining_audio = [f for f in os.listdir(old_dir)
                                               if os.path.splitext(f)[1].lower() in audio_exts]
                            if not remaining_audio:
                                try:
                                    os.remove(old_cover)
                                    print(f"🗑️ [Retag] Removed orphaned cover.jpg from old directory")
                                except Exception:
                                    pass

                    # Cleanup old empty directories
                    transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer'))
                    _cleanup_empty_directories(transfer_dir, current_file_path)

                    # Update DB record
                    db.update_retag_track_path(existing_track['id'], str(new_path))
                    current_file_path = new_path
                else:
                    print(f"📍 [Retag] Path unchanged for '{track_title}', no move needed")
            except Exception as move_err:
                print(f"⚠️ [Retag] Path/move failed for '{track_title}': {move_err}")

            # Download cover art to album directory
            try:
                _download_cover_art(album_info, os.path.dirname(current_file_path))
            except Exception as cover_err:
                print(f"⚠️ [Retag] Cover art download failed: {cover_err}")

            with retag_lock:
                retag_state['processed'] += 1
                retag_state['progress'] = int(retag_state['processed'] / retag_state['total_tracks'] * 100)

        # 5. Update the retag group record with new metadata
        update_kwargs = {
            'artist_name': new_artist.get('name', 'Unknown Artist'),
            'album_name': new_album_name,
            'image_url': new_image_url,
            'total_tracks': total_tracks,
            'release_date': new_release_date
        }
        # Set the correct ID field based on Spotify vs iTunes
        if str(album_id).isdigit():
            update_kwargs['itunes_album_id'] = album_id
            update_kwargs['spotify_album_id'] = None
        else:
            update_kwargs['spotify_album_id'] = album_id
            update_kwargs['itunes_album_id'] = None

        db.update_retag_group(group_id, **update_kwargs)

        with retag_lock:
            retag_state.update({
                "status": "finished",
                "phase": "Retag complete!",
                "progress": 100,
                "current_track": ""
            })
        print(f"✅ [Retag] Retag operation complete for group {group_id}")

    except Exception as e:
        import traceback
        print(f"❌ [Retag] Error during retag: {e}")
        print(traceback.format_exc())
        with retag_lock:
            retag_state.update({
                "status": "error",
                "phase": "Error",
                "error_message": str(e)
            })


def _check_and_remove_from_wishlist(context):
    """
    Check if a successfully downloaded track should be removed from wishlist.
    Extracts Spotify track data from download context and removes from wishlist if found.
    """
    try:
        from core.wishlist_service import get_wishlist_service
        wishlist_service = get_wishlist_service()
        
        # Try to extract Spotify track ID from various sources in the context
        spotify_track_id = None
        
        # Method 1: Direct track_info with id
        track_info = context.get('track_info', {})
        if track_info.get('id'):
            spotify_track_id = track_info['id']
            print(f"📋 [Wishlist] Found Spotify ID from track_info: {spotify_track_id}")
        
        # Method 2: From original search result
        elif context.get('original_search_result', {}).get('id'):
            spotify_track_id = context['original_search_result']['id']
            print(f"📋 [Wishlist] Found Spotify ID from original_search_result: {spotify_track_id}")
        
        # Method 3: Check if this is a wishlist download (context has wishlist_id)
        elif 'wishlist_id' in track_info:
            wishlist_id = track_info['wishlist_id']
            print(f"📋 [Wishlist] Found wishlist_id in context: {wishlist_id}")
            
            # Get the Spotify track ID from the wishlist entry (search all profiles)
            database = get_database()
            all_profiles = database.get_all_profiles()
            wishlist_tracks = []
            for p in all_profiles:
                wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=p['id']))
            for wl_track in wishlist_tracks:
                if wl_track.get('wishlist_id') == wishlist_id:
                    spotify_track_id = wl_track.get('spotify_track_id') or wl_track.get('id')
                    print(f"📋 [Wishlist] Found Spotify ID from wishlist entry: {spotify_track_id}")
                    break

        # Method 4: Try to construct ID from track metadata for fuzzy matching
        if not spotify_track_id:
            track_name = track_info.get('name') or context.get('original_search_result', {}).get('title', '')
            artist_name = _get_track_artist_name(track_info) or _get_track_artist_name(context.get('original_search_result', {}))

            if track_name and artist_name:
                print(f"📋 [Wishlist] No Spotify ID found, checking for fuzzy match: '{track_name}' by '{artist_name}'")

                # Get all wishlist tracks and find potential matches (search all profiles)
                if not wishlist_tracks:
                    database = get_database()
                    all_profiles = database.get_all_profiles()
                    wishlist_tracks = []
                    for p in all_profiles:
                        wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=p['id']))
                for wl_track in wishlist_tracks:
                    wl_name = wl_track.get('name', '').lower()
                    wl_artists = wl_track.get('artists', [])
                    wl_artist_name = ''
                    
                    # Extract artist name from wishlist track
                    if wl_artists:
                        if isinstance(wl_artists[0], dict):
                            wl_artist_name = wl_artists[0].get('name', '').lower()
                        else:
                            wl_artist_name = str(wl_artists[0]).lower()
                    
                    # Simple fuzzy matching
                    if (wl_name == track_name.lower() and wl_artist_name == artist_name.lower()):
                        spotify_track_id = wl_track.get('spotify_track_id') or wl_track.get('id')
                        print(f"📋 [Wishlist] Found fuzzy match - Spotify ID: {spotify_track_id}")
                        break
        
        # If we found a Spotify track ID, remove it from wishlist
        if spotify_track_id:
            print(f"📋 [Wishlist] Attempting to remove track from wishlist: {spotify_track_id}")
            removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True)
            if removed:
                print(f"✅ [Wishlist] Successfully removed track from wishlist: {spotify_track_id}")
            else:
                print(f"ℹ️ [Wishlist] Track not found in wishlist or already removed: {spotify_track_id}")
        else:
            print(f"ℹ️ [Wishlist] No Spotify track ID found for wishlist removal check")
            
    except Exception as e:
        print(f"❌ [Wishlist] Error in wishlist removal check: {e}")
        import traceback
        traceback.print_exc()

def _check_and_remove_track_from_wishlist_by_metadata(track_data):
    """
    Check if a track found during database analysis should be removed from wishlist.
    Uses track metadata (name, artists, id) to find and remove from wishlist.
    """
    try:
        from core.wishlist_service import get_wishlist_service
        wishlist_service = get_wishlist_service()
        
        # Extract track info
        track_name = track_data.get('name', '')
        track_id = track_data.get('id', '')
        artists = track_data.get('artists', [])
        
        print(f"📋 [Analysis] Checking if track should be removed from wishlist: '{track_name}' (ID: {track_id})")
        
        # Method 1: Direct Spotify ID match
        if track_id:
            removed = wishlist_service.mark_track_download_result(track_id, success=True)
            if removed:
                print(f"✅ [Analysis] Removed track from wishlist via direct ID match: {track_id}")
                return True
        
        # Method 2: Fuzzy matching by name and artist if no direct ID match
        if track_name and artists:
            # Extract primary artist name
            primary_artist = ''
            if isinstance(artists[0], dict) and 'name' in artists[0]:
                primary_artist = artists[0]['name']
            elif isinstance(artists[0], str):
                primary_artist = artists[0]
            else:
                primary_artist = str(artists[0])
            
            print(f"📋 [Analysis] No direct ID match, trying fuzzy match: '{track_name}' by '{primary_artist}'")

            # Get all wishlist tracks and find matches (search all profiles)
            database = get_database()
            all_profiles = database.get_all_profiles()
            wishlist_tracks = []
            for p in all_profiles:
                wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=p['id']))
            for wl_track in wishlist_tracks:
                wl_name = wl_track.get('name', '').lower()
                wl_artists = wl_track.get('artists', [])
                wl_artist_name = ''
                
                # Extract artist name from wishlist track
                if wl_artists:
                    if isinstance(wl_artists[0], dict):
                        wl_artist_name = wl_artists[0].get('name', '').lower()
                    else:
                        wl_artist_name = str(wl_artists[0]).lower()
                
                # Fuzzy matching - normalize strings for comparison
                if (wl_name == track_name.lower() and wl_artist_name == primary_artist.lower()):
                    spotify_track_id = wl_track.get('spotify_track_id') or wl_track.get('id')
                    if spotify_track_id:
                        removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True)
                        if removed:
                            print(f"✅ [Analysis] Removed track from wishlist via fuzzy match: {spotify_track_id}")
                            return True
        
        print(f"ℹ️ [Analysis] Track not found in wishlist or already removed: '{track_name}'")
        return False
        
    except Exception as e:
        print(f"❌ [Analysis] Error checking wishlist removal by metadata: {e}")
        import traceback
        traceback.print_exc()
        return False

def _automatic_wishlist_cleanup_after_db_update():
    """
    Automatic wishlist cleanup that runs after database updates.
    This is a simplified version of the cleanup API endpoint designed for background execution.
    """
    try:
        from core.wishlist_service import get_wishlist_service
        from database.music_database import MusicDatabase
        
        wishlist_service = get_wishlist_service()
        db = MusicDatabase()
        active_server = config_manager.get_active_media_server()
        
        print("📋 [Auto Cleanup] Starting automatic wishlist cleanup after database update...")

        # Get all wishlist tracks (across all profiles - cleanup is global)
        database = get_database()
        all_profiles = database.get_all_profiles()
        wishlist_tracks = []
        for p in all_profiles:
            wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=p['id']))
        if not wishlist_tracks:
            print("📋 [Auto Cleanup] No tracks in wishlist to clean up")
            return
        
        print(f"📋 [Auto Cleanup] Found {len(wishlist_tracks)} tracks in wishlist")
        
        removed_count = 0
        
        for track in wishlist_tracks:
            track_name = track.get('name', '')
            artists = track.get('artists', [])
            spotify_track_id = track.get('spotify_track_id') or track.get('id')
            
            # Skip if no essential data
            if not track_name or not artists or not spotify_track_id:
                continue
            
            # Check each artist
            found_in_db = False
            for artist in artists:
                # Handle both string format and dict format
                if isinstance(artist, str):
                    artist_name = artist
                elif isinstance(artist, dict) and 'name' in artist:
                    artist_name = artist['name']
                else:
                    artist_name = str(artist)
                
                try:
                    db_track, confidence = db.check_track_exists(
                        track_name, artist_name, 
                        confidence_threshold=0.7, 
                        server_source=active_server
                    )
                    
                    if db_track and confidence >= 0.7:
                        found_in_db = True
                        print(f"📋 [Auto Cleanup] Track found in database: '{track_name}' by {artist_name} (confidence: {confidence:.2f})")
                        break
                        
                except Exception as db_error:
                    print(f"⚠️ [Auto Cleanup] Error checking database for track '{track_name}': {db_error}")
                    continue
            
            # If found in database, remove from wishlist
            if found_in_db:
                try:
                    removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True)
                    if removed:
                        removed_count += 1
                        print(f"✅ [Auto Cleanup] Removed track from wishlist: '{track_name}' ({spotify_track_id})")
                except Exception as remove_error:
                    print(f"❌ [Auto Cleanup] Error removing track from wishlist: {remove_error}")
        
        print(f"📋 [Auto Cleanup] Completed automatic cleanup: {removed_count} tracks removed from wishlist")
        
    except Exception as e:
        print(f"❌ [Auto Cleanup] Error in automatic wishlist cleanup: {e}")
        import traceback
        traceback.print_exc()

# ── Update detection ─────────────────────────────────────────────
_GITHUB_REPO = "Nezreka/SoulSync"
_update_cache = {'latest_sha': None, 'last_check': 0, 'error': None}
_UPDATE_CHECK_INTERVAL = 3600  # 1 hour

def _get_current_commit_sha():
    """Get the commit SHA of the running instance (env var for Docker, git for local)."""
    # Docker: baked in at build time via COMMIT_SHA build arg
    sha = os.environ.get('SOULSYNC_COMMIT_SHA', '').strip()
    if sha:
        return sha
    # Local dev: read from git
    try:
        import subprocess
        result = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True, text=True, cwd=os.path.dirname(__file__) or '.')
        if result.returncode == 0:
            return result.stdout.strip()
    except Exception:
        pass
    return None

_current_commit_sha = _get_current_commit_sha()

def _check_for_updates():
    """Check GitHub for the latest commit SHA on main branch."""
    import time as _time
    now = _time.time()
    if now - _update_cache['last_check'] < _UPDATE_CHECK_INTERVAL:
        return  # Still fresh
    _update_cache['last_check'] = now
    try:
        import urllib.request
        import json as _json
        req = urllib.request.Request(
            f"https://api.github.com/repos/{_GITHUB_REPO}/commits/main",
            headers={'Accept': 'application/vnd.github.v3+json', 'User-Agent': 'SoulSync-UpdateCheck'}
        )
        with urllib.request.urlopen(req, timeout=10) as resp:
            data = _json.loads(resp.read().decode())
            _update_cache['latest_sha'] = data.get('sha')
            _update_cache['error'] = None
    except Exception as e:
        _update_cache['error'] = str(e)
        logger.debug(f"Update check failed: {e}")

@app.route('/api/update-check', methods=['GET'])
def check_for_update():
    """Check if a newer version is available on GitHub."""
    _check_for_updates()
    current = _current_commit_sha
    latest = _update_cache.get('latest_sha')
    update_available = bool(current and latest and current != latest)
    return jsonify({
        'update_available': update_available,
        'current_sha': current[:8] if current else None,
        'latest_sha': latest[:8] if latest else None,
    })

@app.route('/api/version-info', methods=['GET'])
def get_version_info():
    """
    Returns version information and release notes, matching the GUI's VersionInfoModal content.
    This provides the same data that the GUI version modal displays.
    """
    version_data = {
        "version": "1.8",
        "title": "What's New in SoulSync",
        "subtitle": "Version 1.8 — Latest Changes",
        "sections": [
            {
                "title": "🤖 Automation Engine",
                "description": "Visual drag-and-drop automation builder with 20+ triggers and 14 actions",
                "features": [
                    "• Drag-and-drop builder: connect WHEN triggers → DO actions → NOTIFY alerts",
                    "• Timer triggers: Schedule (interval), Daily Time, Weekly Schedule",
                    "• Event triggers: Track Downloaded, Batch Complete, New Release Found, Playlist Changed, Discovery Complete, and 15 more",
                    "• Actions: Process Wishlist, Scan Watchlist, Refresh/Discover/Sync Playlists, Update Database, Quality Scan, Backup, and more",
                    "• Conditions with match modes (All/Any) and operators (contains, equals, starts_with, not_contains)",
                    "• Configurable delay on actions — wait N minutes after trigger fires before executing",
                    "• System automations for wishlist (every 30 min) and watchlist (every 24 hr) with cross-guards",
                    "• Run Now button on every card for instant testing"
                ]
            },
            {
                "title": "🔍 Playlist Discovery Pipeline",
                "description": "Official Spotify/iTunes metadata enforcement for mirrored playlist sync",
                "features": [
                    "• New 'Discover Playlist' automation action matches raw YouTube/Tidal tracks to official Spotify or iTunes metadata",
                    "• Fuzzy title/artist matching with confidence scoring — minimum 0.7 threshold",
                    "• Discovery results cached globally across playlists for instant repeat lookups",
                    "• Sync Playlist now only includes discovered tracks — undiscovered tracks are skipped entirely",
                    "• Prevents garbage data (wrong artist, no album, no cover art) from reaching the wishlist",
                    "• Spotify-sourced playlists auto-discovered during refresh at confidence 1.0",
                    "• Chain automations: Refresh → Playlist Changed → Discover → Discovery Complete → Sync"
                ]
            },
            {
                "title": "🔔 Notification Integrations",
                "description": "Get alerted when automations run via Discord, Pushbullet, or Telegram",
                "features": [
                    "• Discord Webhook — post automation results to any Discord channel",
                    "• Pushbullet — push notifications to phone and desktop",
                    "• Telegram Bot — send alerts via Telegram Bot API",
                    "• Variable substitution in messages: {artist}, {title}, {album}, {quality}, {time}, and more",
                    "• Attach notifications to any automation — event-based or scheduled"
                ]
            },
            {
                "title": "🔄 YouTube & Tidal Playlist Refresh",
                "description": "Automated re-fetching of mirrored playlists from any source",
                "features": [
                    "• Refresh Mirrored Playlist action now supports YouTube and Tidal alongside Spotify",
                    "• YouTube URLs reconstructed from stored playlist ID for seamless re-parsing",
                    "• Tidal playlists refreshed via Tidal API using stored source ID",
                    "• Discovery data preserved across refreshes — previously matched tracks keep their official metadata",
                    "• Change detection emits Playlist Changed event for automation chaining"
                ]
            },
            {
                "title": "📋 Mirrored Playlists",
                "description": "Persistent cross-service playlist archive on the Sync page",
                "features": [
                    "• Automatically mirrors every parsed playlist from Spotify, Tidal, YouTube, and Beatport",
                    "• New 'Mirrored' tab with source-branded cards showing live discovery and download status",
                    "• Click any mirrored playlist to browse its full track list or run it through discovery",
                    "• Re-parsing the same playlist updates the existing mirror — no duplicates",
                    "• Cards reflect live state: Discovering, Discovered, Downloading, Downloaded",
                    "• Download progress survives page refresh — click a 'Downloading...' card to resume viewing",
                    "• Profile-scoped — each profile has its own mirrored playlists"
                ]
            },
            {
                "title": "👥 Multi-Profile Support",
                "description": "Netflix-style profile picker for shared households",
                "features": [
                    "• Multiple profiles sharing one SoulSync instance with isolated personal data",
                    "• Each profile gets its own watchlist, wishlist, discovery pool, and similar artists",
                    "• Optional PIN protection per profile with admin-only management",
                    "• Profile avatar images via URL with colored-initial fallback",
                    "• Shared music library and service credentials across all profiles",
                    "• Zero-downtime migration — existing data maps to auto-created admin profile",
                    "• Single-user installs see no changes until a second profile is created"
                ]
            },
            {
                "title": "⚡ WebSocket Real-Time Updates",
                "description": "Live push updates replace polling for a faster, more responsive UI",
                "features": [
                    "• Service status, watchlist count, and download progress pushed via WebSocket",
                    "• Dashboard stats, activity feed, and database stats update in real-time",
                    "• Enrichment worker statuses stream live to the sidebar",
                    "• Tool progress (quality scanner, duplicate cleaner, retag, DB update) pushed instantly",
                    "• Automatic HTTP polling fallback when WebSocket is unavailable",
                    "• Profile-scoped WebSocket rooms for per-profile count updates"
                ]
            },
            {
                "title": "🔑 REST API (v1)",
                "description": "Full programmatic access to SoulSync with API key authentication",
                "features": [
                    "• API key authentication with generation and management UI",
                    "• Endpoints for library, watchlist, wishlist, downloads, and system status",
                    "• Reverse proxy support for external access"
                ]
            },
            {
                "title": "👁️ Watchlist Redesign",
                "description": "Enriched artist detail view and improved watchlist management",
                "features": [
                    "• Redesigned watchlist modal with enriched artist detail view",
                    "• Add to watchlist button on each artist in library",
                    "• 'Watch All' button on hero slider to quickly add all displayed artists",
                    "• Add all recommended artists to watchlist in one click",
                    "• View recommended/similar artists panel"
                ]
            },
            {
                "title": "📦 Import Rebuild",
                "description": "Import feature moved to its own dedicated page",
                "features": [
                    "• Rebuilt import feature on a standalone page with improved workflow",
                    "• Import staging guide documentation"
                ]
            },
            {
                "title": "🎨 UI & Customization",
                "description": "Visual improvements and personalization options",
                "features": [
                    "• Custom accent colors with persistent user preference",
                    "• Retag tool layout redesign",
                    "• Watchlist page visual redesign"
                ]
            },
            {
                "title": "🐛 Bug Fixes & Stability",
                "description": "Reliability improvements",
                "features": [
                    "• Fix chromaprint crash (SIGABRT) on 5.1 surround audio files",
                    "• Fix Spotify enrichment worker showing 'Not Authenticated' after re-auth",
                    "• Fix Spotify worker status priority — paused state shown correctly over auth status",
                    "• Fix infinite monitor loop on post-processing completed tasks",
                    "• Fix cancelled downloads not sending album context to wishlist",
                    "• Fix various artist compilations causing AcoustID check failures",
                    "• Fix ListenBrainz database path not respecting DATABASE_PATH env var",
                    "• Fix deleted content not removed during incremental database updates",
                    "• Wishlist always uses 'force download all' mode"
                ]
            }
        ]
    }
    return jsonify(version_data)


def _simple_monitor_task():
    """The actual monitoring task that runs in the background thread."""
    print("🔄 Simple background monitor started")
    last_search_cleanup = 0  # Force initial cleanup on first run
    search_cleanup_interval = 3600  # 1 hour
    initial_cleanup_done = False
    last_download_cleanup = 0
    download_cleanup_interval = 300  # 5 minutes
    
    while True:
        try:
            with matched_context_lock:
                pending_count = len(matched_downloads_context)
            
            if pending_count > 0:
                # Use app_context to safely call endpoint logic from a thread
                with app.app_context():
                    get_download_status()

            # Cleanup stale retry attempts (older than 60 seconds)
            # This prevents memory leaks from stuck/failed downloads
            with _download_retry_lock:
                current_time = time.time()
                stale_keys = [
                    key for key, data in _download_retry_attempts.items()
                    if current_time - data['first_attempt'] > 60
                ]
                for key in stale_keys:
                    print(f"🧹 Cleaning up stale retry attempt: {key}")
                    del _download_retry_attempts[key]

            # Automatic search cleanup every hour (or initial cleanup)
            current_time = time.time()
            should_cleanup = (current_time - last_search_cleanup > search_cleanup_interval) or not initial_cleanup_done
            
            if should_cleanup:
                try:
                    if not initial_cleanup_done:
                        print("🔍 [Auto Cleanup] Performing initial search cleanup in background...")
                        initial_cleanup_done = True
                    else:
                        print("🔍 [Auto Cleanup] Starting scheduled search cleanup...")
                    
                    success = run_async(soulseek_client.maintain_search_history_with_buffer(
                        keep_searches=50, trigger_threshold=200
                    ))
                    if success:
                        cleanup_type = "Initial search history maintenance" if last_search_cleanup == 0 else "Automatic search history maintenance completed"
                        add_activity_item("🧹", "Search Cleanup", cleanup_type, "Now")
                        print("✅ [Auto Cleanup] Search history maintenance completed")
                    else:
                        print("⚠️ [Auto Cleanup] Search history maintenance returned false")
                    last_search_cleanup = current_time
                except Exception as cleanup_error:
                    print(f"❌ [Auto Cleanup] Error in automatic search cleanup: {cleanup_error}")
                    last_search_cleanup = current_time  # Still update to avoid spam
                    initial_cleanup_done = True  # Mark as done even on error to avoid blocking

            # Automatic download cleanup every 5 minutes
            if current_time - last_download_cleanup > download_cleanup_interval:
                try:
                    # Only clear if no batches are actively downloading
                    has_active_batches = False
                    has_post_processing = False
                    with tasks_lock:
                        for batch_data in download_batches.values():
                            if batch_data.get('phase') not in ['complete', 'error', 'cancelled', None]:
                                has_active_batches = True
                                break
                        # Also check for any tasks still in post_processing
                        if not has_active_batches:
                            for task_data in download_tasks.values():
                                if task_data.get('status') == 'post_processing':
                                    has_post_processing = True
                                    break

                    if not has_active_batches:
                        run_async(soulseek_client.clear_all_completed_downloads())
                        # Sweep empty directories left behind by completed downloads
                        if not has_post_processing:
                            _sweep_empty_download_directories()
                        print("✅ [Auto Cleanup] Periodic download cleanup completed")

                    last_download_cleanup = current_time
                except Exception as dl_cleanup_error:
                    print(f"❌ [Auto Cleanup] Error in download cleanup: {dl_cleanup_error}")
                    last_download_cleanup = current_time

            time.sleep(1)
        except Exception as e:
            print(f"❌ Simple monitor error: {e}")
            time.sleep(10)

def start_simple_background_monitor():
    """Starts the simple background monitor thread."""
    monitor_thread = threading.Thread(target=_simple_monitor_task)
    monitor_thread.daemon = True
    monitor_thread.start()

# ===============================
# == AUTOMATIC WISHLIST PROCESSING ==
# ===============================

def _sanitize_track_data_for_processing(track_data):
    """
    Sanitizes track data from wishlist service to ensure consistent format.
    Preserves album dict to retain full metadata (images, id, etc.) and normalizes artist field.
    """
    if not isinstance(track_data, dict):
        print(f"⚠️ [Sanitize] Unexpected track data type: {type(track_data)}")
        return track_data
    
    # Create a copy to avoid modifying original data
    sanitized = track_data.copy()
    
    # Handle album field - preserve dict format to retain full metadata (images, id, etc.)
    # Downstream code already handles both dict and string formats defensively
    raw_album = sanitized.get('album', '')
    if not isinstance(raw_album, (dict, str)):
        sanitized['album'] = str(raw_album)

    # Handle artists field - ensure it's a list of strings
    raw_artists = sanitized.get('artists', [])
    if isinstance(raw_artists, list):
        processed_artists = []
        for artist in raw_artists:
            if isinstance(artist, str):
                processed_artists.append(artist)
            elif isinstance(artist, dict) and 'name' in artist:
                processed_artists.append(artist['name'])
            else:
                processed_artists.append(str(artist))
        sanitized['artists'] = processed_artists
    else:
        print(f"⚠️ [Sanitize] Unexpected artists format: {type(raw_artists)}")
        sanitized['artists'] = [str(raw_artists)] if raw_artists else []
    
    return sanitized

def check_and_recover_stuck_flags():
    """
    Check if wishlist_auto_processing or watchlist_auto_scanning flags are stuck.
    If a flag has been True for more than 2 hours (7200 seconds), reset it.
    This prevents indefinite blocking when processes crash without cleanup.
    """
    global wishlist_auto_processing, wishlist_auto_processing_timestamp
    global watchlist_auto_scanning, watchlist_auto_scanning_timestamp

    import time
    current_time = time.time()
    stuck_timeout = 900  # 15 minutes in seconds (reduced from 2 hours for faster recovery)

    # Check wishlist flag
    if wishlist_auto_processing:
        time_stuck = current_time - wishlist_auto_processing_timestamp
        if time_stuck > stuck_timeout:
            stuck_minutes = time_stuck / 60
            print(f"⚠️ [Stuck Detection] Wishlist auto-processing flag has been stuck for {stuck_minutes:.1f} minutes - RESETTING")
            with wishlist_timer_lock:
                wishlist_auto_processing = False
                wishlist_auto_processing_timestamp = 0

            return True

    # Check watchlist flag
    if watchlist_auto_scanning:
        time_stuck = current_time - watchlist_auto_scanning_timestamp
        if time_stuck > stuck_timeout:
            stuck_minutes = time_stuck / 60
            print(f"⚠️ [Stuck Detection] Watchlist auto-scanning flag has been stuck for {stuck_minutes:.1f} minutes - RESETTING")
            with watchlist_timer_lock:
                watchlist_auto_scanning = False
                watchlist_auto_scanning_timestamp = 0
            return True

    return False

def is_wishlist_actually_processing():
    """
    Check if wishlist is truly processing (not just flag stuck).
    Returns True only if flag is set AND timestamp is recent (< 15 minutes).
    """
    global wishlist_auto_processing, wishlist_auto_processing_timestamp

    if not wishlist_auto_processing:
        return False

    import time
    current_time = time.time()
    time_since_start = current_time - wishlist_auto_processing_timestamp

    # If more than 15 minutes, flag is stuck - auto-recover and return False
    if time_since_start > 900:  # 15 minutes
        stuck_minutes = time_since_start / 60
        print(f"⚠️ [Stuck Detection] Wishlist flag stuck for {stuck_minutes:.1f} minutes - auto-recovering")
        check_and_recover_stuck_flags()
        return False

    return True

def is_watchlist_actually_scanning():
    """
    Check if watchlist is truly scanning (not just flag stuck).
    Returns True only if flag is set AND timestamp is recent (< 15 minutes).
    """
    global watchlist_auto_scanning, watchlist_auto_scanning_timestamp

    if not watchlist_auto_scanning:
        return False

    import time
    current_time = time.time()
    time_since_start = current_time - watchlist_auto_scanning_timestamp

    # If more than 15 minutes, flag is stuck - auto-recover and return False
    if time_since_start > 900:  # 15 minutes
        stuck_minutes = time_since_start / 60
        print(f"⚠️ [Stuck Detection] Watchlist flag stuck for {stuck_minutes:.1f} minutes - auto-recovering")
        check_and_recover_stuck_flags()
        return False

    return True

def _classify_wishlist_track(track):
    """Classify a wishlist track as 'singles' or 'albums'.

    Uses Spotify's album_type as the primary signal (most authoritative),
    falls back to total_tracks heuristic, defaults to 'albums'.

    Returns:
        'singles' or 'albums'
    """
    spotify_data = track.get('spotify_data', {})
    if isinstance(spotify_data, str):
        try:
            import json
            spotify_data = json.loads(spotify_data)
        except Exception:
            spotify_data = {}

    album_data = spotify_data.get('album') or {}
    if not isinstance(album_data, dict):
        album_data = {}
    total_tracks = album_data.get('total_tracks')
    album_type = album_data.get('album_type', '').lower()

    # Prioritize Spotify's album_type classification (most accurate)
    if album_type in ('single', 'ep'):
        return 'singles'
    if album_type in ('album', 'compilation'):
        return 'albums'

    # Fallback: track count heuristic
    if total_tracks is not None and total_tracks > 0:
        return 'singles' if total_tracks < 6 else 'albums'

    # No classification data — default to albums
    return 'albums'


def _process_wishlist_automatically(automation_id=None):
    """Main automatic processing logic that runs in background thread."""
    global wishlist_auto_processing, wishlist_auto_processing_timestamp

    print("🤖 [Auto-Wishlist] Timer triggered - starting automatic wishlist processing...")

    try:
        # CRITICAL FIX: Use smart stuck detection BEFORE acquiring lock
        # This prevents deadlock and handles stuck flags (2-hour timeout)
        if is_wishlist_actually_processing():
            print("⚠️ [Auto-Wishlist] Already processing (verified with stuck detection), skipping.")
            return

        # Check conditions and set flag
        should_skip_already_running = False
        should_skip_watchlist_conflict = False

        with wishlist_timer_lock:
            # Re-check inside lock to handle race conditions
            if wishlist_auto_processing:
                print("⚠️ [Auto-Wishlist] Already processing (race condition check), skipping.")
                should_skip_already_running = True

            # Check if watchlist scan is currently running (using smart detection)
            elif is_watchlist_actually_scanning():
                print("👁️ Watchlist scan in progress, skipping automatic wishlist processing to avoid conflicts.")
                should_skip_watchlist_conflict = True

            else:
                # Set flag and timestamp
                import time
                wishlist_auto_processing = True
                wishlist_auto_processing_timestamp = time.time()
                print(f"🔒 [Auto-Wishlist] Flag set at timestamp {wishlist_auto_processing_timestamp}")

        if should_skip_already_running or should_skip_watchlist_conflict:
            return
        
        # Use app context for database operations
        with app.app_context():
            from core.wishlist_service import get_wishlist_service
            wishlist_service = get_wishlist_service()

            # Check if wishlist has tracks across all profiles
            database = get_database()
            all_profiles = database.get_all_profiles()
            count = sum(wishlist_service.get_wishlist_count(profile_id=p['id']) for p in all_profiles)
            print(f"🔍 [Auto-Wishlist] Wishlist count check: {count} tracks found across {len(all_profiles)} profiles")
            _update_automation_progress(automation_id, progress=10, phase='Checking wishlist',
                                         log_line=f'{count} tracks across {len(all_profiles)} profiles', log_type='info')
            if count == 0:
                print("ℹ️ [Auto-Wishlist] No tracks in wishlist for auto-processing.")
                with wishlist_timer_lock:
                    wishlist_auto_processing = False
                    wishlist_auto_processing_timestamp = 0
                return

            print(f"🎵 [Auto-Wishlist] Found {count} tracks in wishlist, starting automatic processing...")
            
            # Check if wishlist processing is already active (auto or manual)
            playlist_id = "wishlist"
            with tasks_lock:
                for batch_id, batch_data in download_batches.items():
                    batch_playlist_id = batch_data.get('playlist_id')
                    # Check for both auto ('wishlist') and manual ('wishlist_manual') batches
                    if (batch_playlist_id in ['wishlist', 'wishlist_manual'] and
                        batch_data.get('phase') not in ['complete', 'error', 'cancelled']):
                        print(f"⚠️ Wishlist processing already active in another batch ({batch_playlist_id}), skipping automatic start")
                        with wishlist_timer_lock:
                            wishlist_auto_processing = False
                        return

            # CRITICAL: Clean duplicates BEFORE fetching tracks to prevent count mismatches
            # This prevents the "11 tracks shown but 12 counted" bug
            from database.music_database import MusicDatabase
            db = MusicDatabase()

            print("🧹 [Auto-Wishlist] Cleaning duplicate tracks before processing...")
            for p in all_profiles:
                duplicates_removed = db.remove_wishlist_duplicates(profile_id=p['id'])
                if duplicates_removed > 0:
                    print(f"🧹 [Auto-Wishlist] Removed {duplicates_removed} duplicate tracks from profile {p['id']}")

            # CLEANUP: Remove tracks from wishlist that already exist in library
            # This prevents wasting bandwidth on tracks we already have
            print("🧼 [Auto-Wishlist] Checking wishlist against library for already-owned tracks...")
            cleanup_tracks = []
            for p in all_profiles:
                cleanup_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=p['id']))
            cleanup_removed = 0

            for track in cleanup_tracks:
                track_name = track.get('name', '')
                artists = track.get('artists', [])
                spotify_track_id = track.get('spotify_track_id') or track.get('id')

                if not track_name or not artists or not spotify_track_id:
                    continue

                # Check if track exists in library
                found_in_db = False
                for artist in artists:
                    if isinstance(artist, str):
                        artist_name = artist
                    elif isinstance(artist, dict) and 'name' in artist:
                        artist_name = artist['name']
                    else:
                        artist_name = str(artist)

                    try:
                        db_track, confidence = db.check_track_exists(
                            track_name, artist_name,
                            confidence_threshold=0.7,
                            server_source=active_server
                        )

                        if db_track and confidence >= 0.7:
                            found_in_db = True
                            break
                    except Exception as db_error:
                        continue

                # Remove from wishlist if found in library
                if found_in_db:
                    try:
                        removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True)
                        if removed:
                            cleanup_removed += 1
                            print(f"🧼 [Auto-Wishlist] Removed already-owned track: '{track_name}' by {artist_name}")
                    except Exception as remove_error:
                        print(f"⚠️ [Auto-Wishlist] Error removing track from wishlist: {remove_error}")

            if cleanup_removed > 0:
                print(f"✅ [Auto-Wishlist] Cleaned up {cleanup_removed} already-owned tracks from wishlist")
                _update_automation_progress(automation_id, progress=25, phase='Cleaned up duplicates',
                                             log_line=f'Removed {cleanup_removed} already-owned tracks', log_type='success')
            else:
                _update_automation_progress(automation_id, progress=25, phase='Cleanup done',
                                             log_line='No duplicates or already-owned tracks found', log_type='skip')

            # Get wishlist tracks for processing (after cleanup) - combine all profiles
            raw_wishlist_tracks = []
            for p in all_profiles:
                raw_wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=p['id']))
            if not raw_wishlist_tracks:
                print("⚠️ No tracks returned from wishlist service.")
                with wishlist_timer_lock:
                    wishlist_auto_processing = False
                    wishlist_auto_processing_timestamp = 0
                return

            # SANITIZE: Ensure consistent data format from wishlist service
            wishlist_tracks = []
            seen_track_ids_sanitation = set()  # Deduplicate during sanitization
            duplicates_found = 0
            for track in raw_wishlist_tracks:
                sanitized_track = _sanitize_track_data_for_processing(track)
                spotify_track_id = sanitized_track.get('spotify_track_id') or sanitized_track.get('id')

                # Skip duplicates during sanitization
                if spotify_track_id and spotify_track_id in seen_track_ids_sanitation:
                    duplicates_found += 1
                    continue

                wishlist_tracks.append(sanitized_track)
                if spotify_track_id:
                    seen_track_ids_sanitation.add(spotify_track_id)

            if duplicates_found > 0:
                print(f"⚠️ [Auto-Wishlist] Found and removed {duplicates_found} duplicate tracks during sanitization")
            print(f"🔧 [Auto-Wishlist] Sanitized {len(wishlist_tracks)} tracks from wishlist service")

            # CYCLE FILTERING: Get current cycle and filter tracks by category
            with db._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT value FROM metadata WHERE key = 'wishlist_cycle'")
                row = cursor.fetchone()

                if row:
                    current_cycle = row['value']
                else:
                    # Default to albums on first run
                    current_cycle = 'albums'
                    cursor.execute("""
                        INSERT OR REPLACE INTO metadata (key, value, updated_at)
                        VALUES ('wishlist_cycle', 'albums', CURRENT_TIMESTAMP)
                    """)
                    conn.commit()

            # Filter tracks by current cycle category
            filtered_tracks = []
            seen_track_ids_filtering = set()  # Deduplicate during filtering
            for track in wishlist_tracks:
                track_category = _classify_wishlist_track(track)
                spotify_track_id = track.get('spotify_track_id') or track.get('id')
                matches_category = (current_cycle == track_category)

                # Only add if matches category AND not a duplicate
                if matches_category:
                    # Only deduplicate if track has a valid ID
                    if spotify_track_id:
                        if spotify_track_id not in seen_track_ids_filtering:
                            filtered_tracks.append(track)
                            seen_track_ids_filtering.add(spotify_track_id)
                    else:
                        # No ID - can't deduplicate safely, always add
                        filtered_tracks.append(track)

            print(f"🔄 [Auto-Wishlist] Current cycle: {current_cycle}")
            print(f"📊 [Auto-Wishlist] Filtered {len(filtered_tracks)}/{len(wishlist_tracks)} tracks for '{current_cycle}' category")
            _update_automation_progress(automation_id, progress=40, phase=f'Processing {current_cycle}',
                                         log_line=f'Cycle: {current_cycle} — {len(filtered_tracks)} tracks to process', log_type='info')

            # If no tracks in this category, skip to next cycle immediately
            if len(filtered_tracks) == 0:
                print(f"ℹ️ [Auto-Wishlist] No {current_cycle} tracks in wishlist, toggling cycle and scheduling next run")

                # Toggle cycle
                next_cycle = 'singles' if current_cycle == 'albums' else 'albums'
                with db._get_connection() as conn:
                    cursor = conn.cursor()
                    cursor.execute("""
                        INSERT OR REPLACE INTO metadata (key, value, updated_at)
                        VALUES ('wishlist_cycle', ?, CURRENT_TIMESTAMP)
                    """, (next_cycle,))
                    conn.commit()
                print(f"🔄 [Auto-Wishlist] Cycle toggled: {current_cycle} → {next_cycle}")

                with wishlist_timer_lock:
                    wishlist_auto_processing = False
                    wishlist_auto_processing_timestamp = 0
                return

            # Use filtered tracks for processing
            wishlist_tracks = filtered_tracks

            # Create batch for automatic processing
            batch_id = str(uuid.uuid4())
            playlist_name = f"Wishlist (Auto - {current_cycle.capitalize()})"
            
            # Create task queue - convert wishlist tracks to expected format
            with tasks_lock:
                download_batches[batch_id] = {
                    'phase': 'analysis',
                    'playlist_id': playlist_id,
                    'playlist_name': playlist_name,
                    'queue': [],
                    'active_count': 0,
                    'max_concurrent': 1 if current_cycle == 'albums' else 3,  # 1 worker for album source reuse, 3 for singles
                    'queue_index': 0,
                    'analysis_total': len(wishlist_tracks),
                    'analysis_processed': 0,
                    'analysis_results': [],
                    # Track state management (replicating sync.py)
                    'permanently_failed_tracks': [],
                    'cancelled_tracks': set(),
                    # Wishlist tracks are already known-missing — skip the expensive library check
                    'force_download_all': True,
                    # Mark as auto-initiated
                    'auto_initiated': True,
                    'auto_processing_timestamp': time.time(),
                    # Store current cycle for toggling after completion
                    'current_cycle': current_cycle,
                    # Profile context for failed track wishlist re-adds (auto = profile 1 default)
                    'profile_id': 1
                }
            
            print(f"🚀 Starting automatic wishlist batch {batch_id} with {len(wishlist_tracks)} tracks")
            _update_automation_progress(automation_id, progress=50, phase=f'Downloading {len(wishlist_tracks)} tracks',
                                         log_line=f'Started batch: {len(wishlist_tracks)} {current_cycle}', log_type='success')
            
            # Submit the wishlist processing job using existing infrastructure
            missing_download_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, wishlist_tracks)
            
            # Don't mark auto_processing as False here - let completion handler do it
            
    except Exception as e:
        print(f"❌ Error in automatic wishlist processing: {e}")
        import traceback
        traceback.print_exc()
        _update_automation_progress(automation_id, log_line=f'Error: {str(e)}', log_type='error')

        with wishlist_timer_lock:
            wishlist_auto_processing = False
            wishlist_auto_processing_timestamp = 0
        raise  # re-raise so automation wrapper returns error status

# ===============================
# == DATABASE UPDATER API      ==
# ===============================

def _db_update_progress_callback(current_item, processed, total, percentage):
    print(f"📊 [DB Progress] {current_item} - {processed}/{total} ({percentage:.1f}%)")
    with db_update_lock:
        db_update_state.update({
            "current_item": current_item,
            "processed": processed,
            "total": total,
            "progress": percentage
        })

def _db_update_phase_callback(phase):
    print(f"🔄 [DB Phase] {phase}")
    with db_update_lock:
        db_update_state["phase"] = phase

def _db_update_finished_callback(total_artists, total_albums, total_tracks, successful, failed):
    # Check for removal results from the worker
    removed_artists = 0
    removed_albums = 0
    removed_tracks = 0
    if db_update_worker:
        removed_artists = getattr(db_update_worker, 'removed_artists', 0)
        removed_albums = getattr(db_update_worker, 'removed_albums', 0)
        removed_tracks = getattr(db_update_worker, 'removed_tracks', 0)

    removal_msg = ""
    if removed_artists > 0 or removed_albums > 0:
        removal_msg = f" | Removed: {removed_artists} artists, {removed_albums} albums"

    with db_update_lock:
        db_update_state["status"] = "finished"
        db_update_state["phase"] = f"Completed: {successful} successful, {failed} failed{removal_msg}."
        db_update_state["removed_artists"] = removed_artists
        db_update_state["removed_albums"] = removed_albums
        db_update_state["removed_tracks"] = removed_tracks

    # Add activity for database update completion
    summary = f"{total_tracks} tracks, {total_albums} albums, {total_artists} artists processed"
    if removed_artists > 0 or removed_albums > 0:
        summary += f" | {removed_artists} artists, {removed_albums} albums removed"
    add_activity_item("✅", "Database Update Complete", summary, "Now")

    try:
        if automation_engine:
            automation_engine.emit('database_update_completed', {
                'total_artists': str(total_artists),
                'total_albums': str(total_albums),
                'total_tracks': str(total_tracks),
            })
    except Exception:
        pass

    # WISHLIST CLEANUP: Automatically clean up wishlist after database update
    try:
        print("📋 [DB Update] Database update completed, starting automatic wishlist cleanup...")
        # Run cleanup in background to avoid blocking the UI
        missing_download_executor.submit(_automatic_wishlist_cleanup_after_db_update)
    except Exception as cleanup_error:
        print(f"⚠️ [DB Update] Error starting automatic wishlist cleanup: {cleanup_error}")

def _db_update_error_callback(error_message):
    with db_update_lock:
        db_update_state["status"] = "error"
        db_update_state["error_message"] = error_message
    
    # Add activity for database update error
    add_activity_item("❌", "Database Update Failed", error_message, "Now")

def _run_db_update_task(full_refresh, server_type):
    """The actual function that runs in the background thread."""
    global db_update_worker
    media_client = None
    
    if server_type == "plex":
        media_client = plex_client
    elif server_type == "jellyfin":
        media_client = jellyfin_client
    elif server_type == "navidrome":
        media_client = navidrome_client

    if not media_client:
        _db_update_error_callback(f"Media client for '{server_type}' not available.")
        return

    with db_update_lock:
        db_update_worker = DatabaseUpdateWorker(
            media_client=media_client,
            full_refresh=full_refresh,
            server_type=server_type,
            force_sequential=True  # Force sequential processing in web server mode
        )
        # Connect signals to callbacks (handle both Qt and headless modes)
        try:
            # Try Qt signal connection first
            db_update_worker.progress_updated.connect(_db_update_progress_callback)
            db_update_worker.phase_changed.connect(_db_update_phase_callback)
            db_update_worker.finished.connect(_db_update_finished_callback)
            db_update_worker.error.connect(_db_update_error_callback)
        except AttributeError:
            # Headless mode - use callback system
            db_update_worker.connect_callback('progress_updated', _db_update_progress_callback)
            db_update_worker.connect_callback('phase_changed', _db_update_phase_callback)
            db_update_worker.connect_callback('finished', _db_update_finished_callback)
            db_update_worker.connect_callback('error', _db_update_error_callback)

    # This is a blocking call that runs the QThread's logic
    db_update_worker.run()


@app.route('/api/database/stats', methods=['GET'])
def get_database_stats():
    """Endpoint to get current database statistics."""
    try:
        # This logic is adapted from DatabaseStatsWorker
        db = get_database()
        stats = db.get_database_info_for_server()
        return jsonify(stats)
    except Exception as e:
        print(f"Error getting database stats: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/wishlist/count', methods=['GET'])
def get_wishlist_count():
    """Endpoint to get current wishlist count."""
    try:
        from core.wishlist_service import get_wishlist_service
        wishlist_service = get_wishlist_service()
        count = wishlist_service.get_wishlist_count(profile_id=get_current_profile_id())
        return jsonify({"count": count})
    except Exception as e:
        print(f"Error getting wishlist count: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/wishlist/stats', methods=['GET'])
def get_wishlist_stats():
    """
    Get wishlist statistics broken down by category.

    Returns:
        {
            "singles": int,  # Count of singles + EPs
            "albums": int,   # Count of album tracks
            "total": int     # Total count
        }
    """
    try:
        from core.wishlist_service import get_wishlist_service

        wishlist_service = get_wishlist_service()
        raw_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=get_current_profile_id())

        singles_count = 0
        albums_count = 0
        seen_ids = set()

        for track in raw_tracks:
            # Deduplicate by ID (same as tracks endpoint) so counts match
            track_id = track.get('spotify_track_id') or track.get('id')
            if track_id:
                if track_id in seen_ids:
                    continue
                seen_ids.add(track_id)

            category = _classify_wishlist_track(track)
            if category == 'singles':
                singles_count += 1
            else:
                albums_count += 1

        total_count = singles_count + albums_count

        # Calculate time until next auto-processing and get processing state
        next_run_in_seconds = automation_engine.get_system_automation_next_run_seconds('process_wishlist') if automation_engine else 0

        # Use smart function with stuck detection (not raw flag)
        is_processing = is_wishlist_actually_processing()

        # Get current cycle (albums or singles)
        from database.music_database import MusicDatabase
        db = MusicDatabase()
        try:
            with db._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("SELECT value FROM metadata WHERE key = 'wishlist_cycle'")
                row = cursor.fetchone()
                current_cycle = row['value'] if row else 'albums'
        except Exception:
            current_cycle = 'albums'  # Safe fallback

        return jsonify({
            "singles": singles_count,
            "albums": albums_count,
            "total": total_count,
            "next_run_in_seconds": next_run_in_seconds,
            "is_auto_processing": is_processing,
            "current_cycle": current_cycle
        })

    except Exception as e:
        print(f"Error getting wishlist stats: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"error": str(e)}), 500

@app.route('/api/wishlist/cycle', methods=['GET'])
def get_wishlist_cycle():
    """
    Get the current wishlist processing cycle.

    Returns:
        {"cycle": "albums" | "singles"}
    """
    try:
        from database.music_database import MusicDatabase
        db = MusicDatabase()

        # Get cycle from metadata table
        with db._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("SELECT value FROM metadata WHERE key = 'wishlist_cycle'")
            row = cursor.fetchone()

            if row:
                cycle = row['value']
            else:
                # Default to albums on first run
                cycle = 'albums'
                cursor.execute("""
                    INSERT OR REPLACE INTO metadata (key, value, updated_at)
                    VALUES ('wishlist_cycle', 'albums', CURRENT_TIMESTAMP)
                """)
                conn.commit()

        return jsonify({"cycle": cycle})

    except Exception as e:
        print(f"Error getting wishlist cycle: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/wishlist/cycle', methods=['POST'])
def set_wishlist_cycle():
    """
    Set the current wishlist processing cycle.

    Body:
        {"cycle": "albums" | "singles"}
    """
    try:
        data = request.get_json()
        cycle = data.get('cycle')

        if cycle not in ['albums', 'singles']:
            return jsonify({"error": "Invalid cycle. Must be 'albums' or 'singles'"}), 400

        from database.music_database import MusicDatabase
        db = MusicDatabase()

        # Store cycle in metadata table
        with db._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                INSERT OR REPLACE INTO metadata (key, value, updated_at)
                VALUES ('wishlist_cycle', ?, CURRENT_TIMESTAMP)
            """, (cycle,))
            conn.commit()

        print(f"✅ Wishlist cycle set to: {cycle}")
        return jsonify({"success": True, "cycle": cycle})

    except Exception as e:
        print(f"Error setting wishlist cycle: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/discovery/lookback-period', methods=['GET'])
def get_discovery_lookback_period():
    """
    Get the discovery pool lookback period setting.

    Returns:
        {"period": "7" | "30" | "90" | "180" | "all"}
    """
    try:
        from database.music_database import MusicDatabase
        db = MusicDatabase()

        # Get lookback period from metadata table
        with db._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("SELECT value FROM metadata WHERE key = 'discovery_lookback_period'")
            row = cursor.fetchone()

            if row:
                period = row['value']
            else:
                # Default to 30 days on first access
                period = '30'
                cursor.execute("""
                    INSERT OR REPLACE INTO metadata (key, value, updated_at)
                    VALUES ('discovery_lookback_period', '30', CURRENT_TIMESTAMP)
                """)
                conn.commit()

        return jsonify({"period": period})

    except Exception as e:
        print(f"Error getting discovery lookback period: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/discovery/lookback-period', methods=['POST'])
def set_discovery_lookback_period():
    """
    Set the discovery pool lookback period setting.

    Body:
        {"period": "7" | "30" | "90" | "180" | "all"}
    """
    try:
        data = request.get_json()
        period = data.get('period')

        valid_periods = ['7', '30', '90', '180', 'all']
        if period not in valid_periods:
            return jsonify({"error": f"Invalid period. Must be one of: {', '.join(valid_periods)}"}), 400

        from database.music_database import MusicDatabase
        db = MusicDatabase()

        # Store lookback period in metadata table
        with db._get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                INSERT OR REPLACE INTO metadata (key, value, updated_at)
                VALUES ('discovery_lookback_period', ?, CURRENT_TIMESTAMP)
            """, (period,))

            # When expanding the lookback window (especially to "entire disco"),
            # reset scan timestamps so the next scan re-discovers older releases
            # that were filtered out under the previous narrower setting
            cursor.execute("UPDATE watchlist_artists SET last_scan_timestamp = NULL")
            reset_count = cursor.rowcount
            conn.commit()

        print(f"✅ Discovery lookback period set to: {period}, reset scan timestamps on {reset_count} artists")
        return jsonify({"success": True, "period": period})

    except Exception as e:
        print(f"Error setting discovery lookback period: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/wishlist/tracks', methods=['GET'])
def get_wishlist_tracks():
    """
    Endpoint to get wishlist tracks for display in modal.
    Supports category filtering via query parameter.

    Query Parameters:
        category (optional): 'singles' or 'albums' - filters tracks by album type
        limit (optional): Maximum number of tracks to return (for performance)
    """
    try:
        from core.wishlist_service import get_wishlist_service
        from database.music_database import MusicDatabase

        # Get category filter and limit from query params
        category = request.args.get('category', None)  # None = all tracks
        limit = request.args.get('limit', type=int, default=None)  # None = no limit

        # Clean duplicates ONLY if no active wishlist download is running
        # This prevents count mismatches during active downloads
        with tasks_lock:
            wishlist_batch_active = any(
                batch.get('playlist_id') == 'wishlist' and batch.get('phase') in ['analysis', 'downloading']
                for batch in download_batches.values()
            )

        if not wishlist_batch_active:
            db = MusicDatabase()
            duplicates_removed = db.remove_wishlist_duplicates(profile_id=get_current_profile_id())
            if duplicates_removed > 0:
                print(f"🧹 Cleaned {duplicates_removed} duplicate tracks from wishlist")
        else:
            print(f"⏸️ Skipping wishlist duplicate cleanup - download in progress")

        wishlist_service = get_wishlist_service()
        raw_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=get_current_profile_id())

        # SANITIZE: Ensure consistent data format for frontend
        sanitized_tracks = []
        seen_track_ids_sanitation = set()  # Deduplicate during sanitization
        duplicates_found = 0
        for track in raw_tracks:
            sanitized_track = _sanitize_track_data_for_processing(track)
            spotify_track_id = sanitized_track.get('spotify_track_id') or sanitized_track.get('id')

            # Skip duplicates during sanitization
            if spotify_track_id and spotify_track_id in seen_track_ids_sanitation:
                duplicates_found += 1
                continue

            sanitized_tracks.append(sanitized_track)
            if spotify_track_id:
                seen_track_ids_sanitation.add(spotify_track_id)

        if duplicates_found > 0:
            print(f"⚠️ [API-Wishlist-Tracks] Found and removed {duplicates_found} duplicate tracks during sanitization")

        # FILTER by category if specified
        if category:
            filtered_tracks = []
            seen_track_ids_filtering = set()  # Deduplicate during filtering
            for track in sanitized_tracks:
                track_category = _classify_wishlist_track(track)
                spotify_track_id = track.get('spotify_track_id') or track.get('id')
                matches_category = (category == track_category)

                # Only add if matches category AND not a duplicate
                if matches_category:
                    # Only deduplicate if track has a valid ID
                    if spotify_track_id:
                        if spotify_track_id not in seen_track_ids_filtering:
                            filtered_tracks.append(track)
                            seen_track_ids_filtering.add(spotify_track_id)
                    else:
                        # No ID - can't deduplicate safely, always add
                        filtered_tracks.append(track)

                # Apply limit early for performance
                if limit and len(filtered_tracks) >= limit:
                    break

            print(f"📊 Wishlist filter: {len(filtered_tracks)}/{len(sanitized_tracks)} tracks in '{category}' category (limit: {limit or 'none'})")
            return jsonify({"tracks": filtered_tracks, "category": category})

        # Apply limit to non-filtered results
        result_tracks = sanitized_tracks[:limit] if limit else sanitized_tracks
        return jsonify({"tracks": result_tracks})
    except Exception as e:
        print(f"Error getting wishlist tracks: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/wishlist/download_missing', methods=['POST'])
def start_wishlist_missing_downloads():
    """
    This endpoint fetches wishlist tracks and manages them with batch processing
    identical to playlist processing, maintaining exactly 3 concurrent downloads.
    """
    try:
        # Check if auto-processing is currently running (prevent concurrent wishlist access)
        if is_wishlist_actually_processing():
            return jsonify({
                "error": "Wishlist auto-processing is currently running. Please wait for it to complete.",
                "retry_after": 30
            }), 409

        data = request.get_json() or {}
        force_download_all = data.get('force_download_all', False)
        category = data.get('category')  # Get category filter (albums or singles)
        track_ids = data.get('track_ids')  # NEW: Get specific track IDs from frontend

        from core.wishlist_service import get_wishlist_service
        from database.music_database import MusicDatabase

        wishlist_service = get_wishlist_service()

        # CRITICAL: Clean duplicates BEFORE fetching tracks to prevent count mismatches
        # This prevents the "11 tracks shown but 12 counted" bug
        print("🧹 [Manual-Wishlist] Cleaning duplicate tracks before download...")
        db = MusicDatabase()
        manual_profile_id = get_current_profile_id()
        duplicates_removed = db.remove_wishlist_duplicates(profile_id=manual_profile_id)
        if duplicates_removed > 0:
            print(f"🧹 [Manual-Wishlist] Removed {duplicates_removed} duplicate tracks")

        # CLEANUP: Remove tracks from wishlist that already exist in library
        # This prevents wasting bandwidth on tracks we already have
        print("🧼 [Manual-Wishlist] Checking wishlist against library for already-owned tracks...")
        cleanup_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=manual_profile_id)
        cleanup_removed = 0

        for track in cleanup_tracks:
            track_name = track.get('name', '')
            artists = track.get('artists', [])
            spotify_track_id = track.get('spotify_track_id') or track.get('id')

            if not track_name or not artists or not spotify_track_id:
                continue

            # Check if track exists in library
            found_in_db = False
            for artist in artists:
                if isinstance(artist, str):
                    artist_name = artist
                elif isinstance(artist, dict) and 'name' in artist:
                    artist_name = artist['name']
                else:
                    artist_name = str(artist)

                try:
                    db_track, confidence = db.check_track_exists(
                        track_name, artist_name,
                        confidence_threshold=0.7,
                        server_source=active_server
                    )

                    if db_track and confidence >= 0.7:
                        found_in_db = True
                        break
                except Exception as db_error:
                    continue

            # Remove from wishlist if found in library
            if found_in_db:
                try:
                    removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True)
                    if removed:
                        cleanup_removed += 1
                        print(f"🧼 [Manual-Wishlist] Removed already-owned track: '{track_name}' by {artist_name}")
                except Exception as remove_error:
                    print(f"⚠️ [Manual-Wishlist] Error removing track from wishlist: {remove_error}")

        if cleanup_removed > 0:
            print(f"✅ [Manual-Wishlist] Cleaned up {cleanup_removed} already-owned tracks from wishlist")

        # Get wishlist tracks formatted for download modal (after cleanup)
        raw_wishlist_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=manual_profile_id)
        if not raw_wishlist_tracks:
            return jsonify({"success": False, "error": "No tracks in wishlist"}), 400

        # SANITIZE: Ensure consistent data format from wishlist service
        wishlist_tracks = []
        seen_track_ids_sanitation = set()  # Deduplicate during sanitization
        duplicates_found = 0
        for track in raw_wishlist_tracks:
            sanitized_track = _sanitize_track_data_for_processing(track)
            spotify_track_id = sanitized_track.get('spotify_track_id') or sanitized_track.get('id')

            # Skip duplicates during sanitization
            if spotify_track_id and spotify_track_id in seen_track_ids_sanitation:
                duplicates_found += 1
                continue

            wishlist_tracks.append(sanitized_track)
            if spotify_track_id:
                seen_track_ids_sanitation.add(spotify_track_id)

        if duplicates_found > 0:
            print(f"⚠️ [Manual-Wishlist] Found and removed {duplicates_found} duplicate tracks during sanitization")
        print(f"🔧 [Manual-Wishlist] Sanitized {len(wishlist_tracks)} tracks from wishlist service")

        # FILTER BY TRACK IDs if specified (prioritized - prevents race conditions)
        if track_ids:
            # Build a lookup by track ID for O(1) access
            track_lookup = {}
            for track in wishlist_tracks:
                spotify_track_id = track.get('spotify_track_id') or track.get('id')
                if spotify_track_id and spotify_track_id not in track_lookup:
                    track_lookup[spotify_track_id] = track

            # Iterate in track_ids order (matches frontend display order)
            # so that enumerate()-based track_index aligns with data-track-index in the modal
            filtered_tracks = []
            seen_track_ids = set()
            for tid in track_ids:
                if tid in track_lookup and tid not in seen_track_ids:
                    filtered_tracks.append(track_lookup[tid])
                    seen_track_ids.add(tid)

            wishlist_tracks = filtered_tracks
            print(f"🎯 [Manual-Wishlist] Filtered to {len(wishlist_tracks)} specific tracks by ID (preserving frontend display order)")

        # FILTER BY CATEGORY if specified and no track_ids (backward compatibility)
        elif category:
            import json
            filtered_tracks = []
            seen_track_ids = set()  # Track IDs we've already added to prevent duplicates
            for track in wishlist_tracks:
                # Extract track count from spotify_data
                spotify_data = track.get('spotify_data', {})
                if isinstance(spotify_data, str):
                    try:
                        spotify_data = json.loads(spotify_data)
                    except:
                        spotify_data = {}

                album_data = spotify_data.get('album', {})
                total_tracks = album_data.get('total_tracks')
                album_type = album_data.get('album_type', 'album').lower()

                # Categorize by track count if available, otherwise use album_type
                # Single: 1 track, EP: 2-5 tracks, Album: 6+ tracks
                is_single_or_ep = False
                is_album = False

                if total_tracks is not None and total_tracks > 0:
                    # Use track count (most accurate)
                    is_single_or_ep = total_tracks < 6
                    is_album = total_tracks >= 6
                else:
                    # Fall back to Spotify's album_type
                    is_single_or_ep = album_type in ['single', 'ep']
                    is_album = album_type == 'album'

                spotify_track_id = track.get('spotify_track_id') or track.get('id')
                matches_category = (category == 'singles' and is_single_or_ep) or (category == 'albums' and is_album)

                # Only add if matches category AND not a duplicate
                if matches_category:
                    # Only deduplicate if track has a valid ID
                    if spotify_track_id:
                        if spotify_track_id not in seen_track_ids:
                            filtered_tracks.append(track)
                            seen_track_ids.add(spotify_track_id)
                    else:
                        # No ID - can't deduplicate safely, always add
                        filtered_tracks.append(track)

            wishlist_tracks = filtered_tracks
            print(f"🔍 [Manual-Wishlist] Filtered to {len(wishlist_tracks)} tracks for category: {category}")

        # Add activity for wishlist download start
        add_activity_item("📥", "Wishlist Download Started", f"{len(wishlist_tracks)} tracks", "Now")

        batch_id = str(uuid.uuid4())
        
        # Use "wishlist" as the playlist_id for consistency in the modal system
        playlist_id = "wishlist"
        playlist_name = "Wishlist"
        
        # Create task queue for this batch - convert wishlist tracks to the expected format
        task_queue = []
        with tasks_lock:
            download_batches[batch_id] = {
                'phase': 'analysis',
                'playlist_id': playlist_id,
                'playlist_name': playlist_name,
                'queue': task_queue,
                'active_count': 0,
                'max_concurrent': 1 if category == 'albums' else 3,  # 1 worker for album source reuse, 3 for singles
                'queue_index': 0,
                'analysis_total': len(wishlist_tracks),
                'analysis_processed': 0,
                'analysis_results': [],
                # Track state management (replicating sync.py)
                'permanently_failed_tracks': [],
                'cancelled_tracks': set(),
                # Wishlist tracks are already known-missing — always skip the library check
                'force_download_all': True,
                # Profile context for failed track wishlist re-adds
                'profile_id': manual_profile_id
            }

        # Submit the wishlist processing job using the same processing function
        missing_download_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, wishlist_tracks)

        return jsonify({
            "success": True,
            "batch_id": batch_id
        })
        
    except Exception as e:
        print(f"Error starting wishlist download process: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/wishlist/clear', methods=['POST'])
def clear_wishlist():
    """Endpoint to clear all tracks from the wishlist."""
    try:
        from core.wishlist_service import get_wishlist_service
        wishlist_service = get_wishlist_service()
        success = wishlist_service.clear_wishlist(profile_id=get_current_profile_id())

        if success:
            return jsonify({"success": True, "message": "Wishlist cleared successfully"})
        else:
            return jsonify({"success": False, "error": "Failed to clear wishlist"}), 500
            
    except Exception as e:
        print(f"Error clearing wishlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/wishlist/cleanup', methods=['POST'])
def cleanup_wishlist():
    """Endpoint to remove tracks from wishlist that already exist in the database."""
    try:
        from core.wishlist_service import get_wishlist_service
        from database.music_database import MusicDatabase
        
        wishlist_service = get_wishlist_service()
        db = MusicDatabase()
        active_server = config_manager.get_active_media_server()
        
        print("📋 [Wishlist Cleanup] Starting wishlist cleanup process...")

        # Get wishlist tracks for current profile
        cleanup_profile_id = get_current_profile_id()
        wishlist_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=cleanup_profile_id)
        if not wishlist_tracks:
            return jsonify({"success": True, "message": "No tracks in wishlist to clean up", "removed_count": 0})

        print(f"📋 [Wishlist Cleanup] Found {len(wishlist_tracks)} tracks in wishlist")
        
        removed_count = 0
        processed_count = 0
        
        for track in wishlist_tracks:
            processed_count += 1
            track_name = track.get('name', '')
            artists = track.get('artists', [])
            spotify_track_id = track.get('spotify_track_id') or track.get('id')
            
            # Skip if no essential data
            if not track_name or not artists or not spotify_track_id:
                continue
            
            print(f"📋 [Wishlist Cleanup] Checking track {processed_count}/{len(wishlist_tracks)}: '{track_name}'")
            
            # Check each artist
            found_in_db = False
            for artist in artists:
                # Handle both string format and dict format
                if isinstance(artist, str):
                    artist_name = artist
                elif isinstance(artist, dict) and 'name' in artist:
                    artist_name = artist['name']
                else:
                    artist_name = str(artist)
                
                try:
                    db_track, confidence = db.check_track_exists(
                        track_name, artist_name, 
                        confidence_threshold=0.7, 
                        server_source=active_server
                    )
                    
                    if db_track and confidence >= 0.7:
                        found_in_db = True
                        print(f"📋 [Wishlist Cleanup] Track found in database: '{track_name}' by {artist_name} (confidence: {confidence:.2f})")
                        break
                        
                except Exception as db_error:
                    print(f"⚠️ [Wishlist Cleanup] Error checking database for track '{track_name}': {db_error}")
                    continue
            
            # If found in database, remove from wishlist
            if found_in_db:
                try:
                    removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True)
                    if removed:
                        removed_count += 1
                        print(f"✅ [Wishlist Cleanup] Removed track from wishlist: '{track_name}' ({spotify_track_id})")
                    else:
                        print(f"⚠️ [Wishlist Cleanup] Failed to remove track from wishlist: '{track_name}' ({spotify_track_id})")
                except Exception as remove_error:
                    print(f"❌ [Wishlist Cleanup] Error removing track from wishlist: {remove_error}")
        
        print(f"📋 [Wishlist Cleanup] Completed cleanup: {removed_count} tracks removed from wishlist")
        
        return jsonify({
            "success": True, 
            "message": f"Wishlist cleanup completed: {removed_count} tracks removed",
            "removed_count": removed_count,
            "processed_count": processed_count
        })
        
    except Exception as e:
        print(f"Error in wishlist cleanup: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/wishlist/remove-track', methods=['POST'])
def remove_track_from_wishlist():
    """Endpoint to remove a single track from the wishlist."""
    try:
        from core.wishlist_service import get_wishlist_service

        data = request.get_json()
        spotify_track_id = data.get('spotify_track_id')

        if not spotify_track_id:
            return jsonify({"success": False, "error": "No spotify_track_id provided"}), 400

        wishlist_service = get_wishlist_service()
        success = wishlist_service.remove_track_from_wishlist(spotify_track_id, profile_id=get_current_profile_id())

        if success:
            logger.info(f"Successfully removed track from wishlist: {spotify_track_id}")
            return jsonify({"success": True, "message": "Track removed from wishlist"})
        else:
            logger.warning(f"Failed to remove track from wishlist: {spotify_track_id}")
            return jsonify({"success": False, "error": "Track not found in wishlist"}), 404

    except Exception as e:
        logger.error(f"Error removing track from wishlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/wishlist/remove-album', methods=['POST'])
def remove_album_from_wishlist():
    """Endpoint to remove all tracks from an album from the wishlist."""
    try:
        from core.wishlist_service import get_wishlist_service
        import json

        data = request.get_json()
        album_id = data.get('album_id')

        if not album_id:
            return jsonify({"success": False, "error": "No album_id provided"}), 400

        wishlist_service = get_wishlist_service()
        all_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=get_current_profile_id())

        # Find all tracks that belong to this album
        tracks_to_remove = []
        for track in all_tracks:
            spotify_data = track.get('spotify_data', {})
            if isinstance(spotify_data, str):
                try:
                    spotify_data = json.loads(spotify_data)
                except:
                    spotify_data = {}

            # Get album ID - safely handle null album data
            album_data = spotify_data.get('album') or {}
            if not isinstance(album_data, dict):
                album_data = {}
            track_album_id = album_data.get('id')

            if not track_album_id:
                # Create custom ID matching frontend logic exactly
                # album_data is guaranteed to be a dict from above check
                album_name = album_data.get('name', 'Unknown Album')
                artists = spotify_data.get('artists', [])
                if artists and isinstance(artists[0], dict):
                    artist_name = artists[0].get('name', 'Unknown Artist')
                elif artists and isinstance(artists[0], str):
                    artist_name = artists[0]
                else:
                    artist_name = 'Unknown Artist'
                custom_id = f"{album_name}_{artist_name}"
                # Match frontend regex exactly:
                # 1. Remove all special chars except spaces, underscores, hyphens: /[^a-zA-Z0-9\s_-]/g
                # 2. Replace consecutive whitespace with single underscore: /\s+/g
                track_album_id = re.sub(r'[^a-zA-Z0-9\s_-]', '', custom_id)  # Remove special chars
                track_album_id = re.sub(r'\s+', '_', track_album_id).lower()  # Replace spaces & lowercase

            # Match by album ID
            if track_album_id == album_id:
                spotify_track_id = track.get('spotify_track_id') or track.get('id')
                if spotify_track_id:
                    tracks_to_remove.append(spotify_track_id)

        # Remove all matching tracks
        removed_count = 0
        album_remove_pid = get_current_profile_id()
        for spotify_track_id in tracks_to_remove:
            if wishlist_service.remove_track_from_wishlist(spotify_track_id, profile_id=album_remove_pid):
                removed_count += 1

        if removed_count > 0:
            logger.info(f"Successfully removed {removed_count} tracks from album {album_id}")
            return jsonify({
                "success": True,
                "message": f"Removed {removed_count} track(s) from wishlist",
                "removed_count": removed_count
            })
        else:
            logger.warning(f"No tracks found for album {album_id}")
            return jsonify({"success": False, "error": "No tracks found for this album"}), 404

    except Exception as e:
        logger.error(f"Error removing album from wishlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/wishlist/remove-batch', methods=['POST'])
def remove_batch_from_wishlist():
    """Endpoint to remove multiple tracks from the wishlist."""
    try:
        from core.wishlist_service import get_wishlist_service

        data = request.get_json()
        spotify_track_ids = data.get('spotify_track_ids', [])

        if not spotify_track_ids or not isinstance(spotify_track_ids, list):
            return jsonify({"success": False, "error": "Missing or invalid spotify_track_ids"}), 400

        wishlist_service = get_wishlist_service()
        removed = 0
        pid = get_current_profile_id()
        for track_id in spotify_track_ids:
            if wishlist_service.remove_track_from_wishlist(track_id, profile_id=pid):
                removed += 1

        logger.info(f"Batch removed {removed} track(s) from wishlist")
        return jsonify({
            "success": True,
            "removed": removed,
            "message": f"Removed {removed} track{'s' if removed != 1 else ''} from wishlist"
        })

    except Exception as e:
        logger.error(f"Error batch removing from wishlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/add-album-to-wishlist', methods=['POST'])
def add_album_track_to_wishlist():
    """Endpoint to add a single track from an album to the wishlist."""
    try:
        from core.wishlist_service import get_wishlist_service

        data = request.get_json()
        if not data:
            return jsonify({"success": False, "error": "No data provided"}), 400

        track = data.get('track')
        artist = data.get('artist')
        album = data.get('album')
        source_type = data.get('source_type', 'album')
        source_context = data.get('source_context', {})

        if not track or not artist or not album:
            return jsonify({"success": False, "error": "Missing required fields: track, artist, album"}), 400

        # Create Spotify track data format expected by wishlist service
        # Handle both formats: Spotify API format (images array) and library format (image_url string)
        album_images = []
        if 'images' in album and album.get('images'):
            # Spotify API format with images array
            album_images = album['images']
        elif 'image_url' in album and album.get('image_url'):
            # Library format with single image_url - convert to Spotify format
            album_images = [{'url': album['image_url'], 'height': 640, 'width': 640}]

        spotify_track_data = {
            'id': track.get('id'),
            'name': track.get('name'),
            'artists': track.get('artists', []),
            'album': {
                'id': album.get('id'),
                'name': album.get('name'),
                'artists': album.get('artists', []),
                'images': album_images,
                'album_type': album.get('album_type', 'album'),
                'release_date': album.get('release_date', ''),
                'total_tracks': album.get('total_tracks', 1)
            },
            'duration_ms': track.get('duration_ms', 0),
            'track_number': track.get('track_number', 1),
            'disc_number': track.get('disc_number', 1),
            'explicit': track.get('explicit', False),
            'popularity': track.get('popularity', 0),
            'preview_url': track.get('preview_url'),
            'external_urls': track.get('external_urls', {})
        }

        # Add source context information
        enhanced_source_context = {
            **source_context,
            'artist_id': artist.get('id'),
            'artist_name': artist.get('name'),
            'album_id': album.get('id'),
            'album_name': album.get('name'),
            'added_via': 'library_wishlist_modal'
        }

        # Get wishlist service and add track
        wishlist_service = get_wishlist_service()

        success = wishlist_service.add_spotify_track_to_wishlist(
            spotify_track_data=spotify_track_data,
            failure_reason="Added from library (incomplete album)",
            source_type=source_type,
            source_context=enhanced_source_context,
            profile_id=get_current_profile_id()
        )

        if success:
            print(f"✅ Added track '{track.get('name')}' by '{artist.get('name')}' to wishlist")
            return jsonify({
                "success": True,
                "message": f"Added '{track.get('name')}' to wishlist"
            })
        else:
            print(f"❌ Failed to add track '{track.get('name')}' to wishlist")
            return jsonify({
                "success": False,
                "error": "Failed to add track to wishlist"
            })

    except Exception as e:
        print(f"❌ Error adding track to wishlist: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/database/update', methods=['POST'])
def start_database_update():
    """Endpoint to start the database update process."""
    global db_update_worker
    with db_update_lock:
        if db_update_state["status"] == "running":
            return jsonify({"success": False, "error": "An update is already in progress."}), 409

        data = request.get_json()
        full_refresh = data.get('full_refresh', False)
        active_server = config_manager.get_active_media_server()

        db_update_state.update({
            "status": "running",
            "phase": "Initializing...",
            "progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": ""
        })
        
        # Add activity for database update start
        update_type = "Full" if full_refresh else "Incremental"
        server_name = active_server.capitalize()
        add_activity_item("🗄️", "Database Update", f"Starting {update_type.lower()} update from {server_name}...", "Now")
        
        # Submit the worker function to the executor
        db_update_executor.submit(_run_db_update_task, full_refresh, active_server)

    return jsonify({"success": True, "message": "Database update started."})

@app.route('/api/database/update/status', methods=['GET'])
def get_database_update_status():
    """Endpoint to poll for the current update status."""
    with db_update_lock:
        # Debug: Log current state occasionally
        if db_update_state["status"] == "running":
            print(f"📊 [Status Check] {db_update_state['processed']}/{db_update_state['total']} ({db_update_state['progress']:.1f}%) - {db_update_state['phase']}")
        return jsonify(db_update_state)

@app.route('/api/database/update/stop', methods=['POST'])
def stop_database_update():
    """Endpoint to stop the current database update."""
    global db_update_worker
    with db_update_lock:
        if db_update_worker and db_update_state["status"] == "running":
            db_update_worker.stop()
            db_update_state["status"] = "finished"
            db_update_state["phase"] = "Update stopped by user."
            return jsonify({"success": True, "message": "Stop request sent."})
        else:
            return jsonify({"success": False, "error": "No update is currently running."}), 404

@app.route('/api/database/backup', methods=['POST'])
def backup_database_endpoint():
    """Create a rolling backup of the database (max 3)."""
    try:
        import sqlite3, glob as _glob
        db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db')
        if not os.path.exists(db_path):
            return jsonify({"success": False, "error": "Database file not found"}), 404
        max_backups = 3
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        backup_path = f"{db_path}.backup_{timestamp}"
        src = sqlite3.connect(db_path)
        dst = sqlite3.connect(backup_path)
        src.backup(dst)
        dst.close()
        src.close()
        size_mb = round(os.path.getsize(backup_path) / (1024 * 1024), 1)
        # Rolling cleanup
        existing = sorted(_glob.glob(f"{db_path}.backup_*"), key=os.path.getmtime)
        while len(existing) > max_backups:
            try:
                os.remove(existing.pop(0))
            except Exception:
                pass
        return jsonify({"success": True, "backup_path": backup_path, "size_mb": size_mb})
    except Exception as e:
        return jsonify({"success": False, "error": str(e)}), 500

# ===============================
# == QUALITY SCANNER           ==
# ===============================

# Quality tier mappings
QUALITY_TIERS = {
    'lossless': {
        'extensions': ['.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif'],
        'tier': 1
    },
    'high_lossy': {
        'extensions': ['.opus', '.ogg'],
        'tier': 2
    },
    'standard_lossy': {
        'extensions': ['.m4a', '.aac'],
        'tier': 3
    },
    'low_lossy': {
        'extensions': ['.mp3', '.wma'],
        'tier': 4
    }
}

def _get_quality_tier_from_extension(file_path):
    """Determine quality tier from file extension"""
    if not file_path:
        return ('unknown', 999)

    ext = os.path.splitext(file_path)[1].lower()

    for tier_name, tier_data in QUALITY_TIERS.items():
        if ext in tier_data['extensions']:
            return (tier_name, tier_data['tier'])

    return ('unknown', 999)

def _run_quality_scanner(scope='watchlist', profile_id=1):
    """Main quality scanner worker function"""
    from core.wishlist_service import get_wishlist_service
    from database.music_database import MusicDatabase

    try:
        with quality_scanner_lock:
            quality_scanner_state["status"] = "running"
            quality_scanner_state["phase"] = "Initializing scan..."
            quality_scanner_state["progress"] = 0
            quality_scanner_state["processed"] = 0
            quality_scanner_state["total"] = 0
            quality_scanner_state["quality_met"] = 0
            quality_scanner_state["low_quality"] = 0
            quality_scanner_state["matched"] = 0
            quality_scanner_state["results"] = []
            quality_scanner_state["error_message"] = ""

        print(f"🔍 [Quality Scanner] Starting scan with scope: {scope}")

        # Get database instance
        db = MusicDatabase()

        # Get quality profile to determine preferred quality
        quality_profile = db.get_quality_profile()
        preferred_qualities = quality_profile.get('qualities', {})

        # Determine minimum acceptable tier based on enabled qualities
        min_acceptable_tier = 999
        for quality_name, quality_config in preferred_qualities.items():
            if quality_config.get('enabled', False):
                # Map quality profile names to tier names
                tier_map = {
                    'flac': 'lossless',
                    'mp3_320': 'low_lossy',
                    'mp3_256': 'low_lossy',
                    'mp3_192': 'low_lossy'
                }
                tier_name = tier_map.get(quality_name)
                if tier_name:
                    tier_num = QUALITY_TIERS[tier_name]['tier']
                    min_acceptable_tier = min(min_acceptable_tier, tier_num)

        print(f"🎵 [Quality Scanner] Minimum acceptable tier: {min_acceptable_tier}")

        # Get tracks to scan based on scope
        with quality_scanner_lock:
            quality_scanner_state["phase"] = "Loading tracks from database..."

        if scope == 'watchlist':
            # Get watchlist artists
            watchlist_artists = db.get_watchlist_artists(profile_id=profile_id)
            if not watchlist_artists:
                with quality_scanner_lock:
                    quality_scanner_state["status"] = "finished"
                    quality_scanner_state["phase"] = "No watchlist artists found"
                    quality_scanner_state["error_message"] = "Please add artists to watchlist first"
                print(f"⚠️ [Quality Scanner] No watchlist artists found")
                return

            # Get artist names from watchlist
            artist_names = [artist.artist_name for artist in watchlist_artists]
            print(f"📋 [Quality Scanner] Scanning {len(artist_names)} watchlist artists")

            # Get all tracks for these artists by name
            conn = db._get_connection()
            placeholders = ','.join(['?' for _ in artist_names])
            tracks_to_scan = conn.execute(
                f"SELECT t.id, t.title, t.artist_id, t.album_id, t.file_path, t.bitrate, a.name as artist_name, al.title as album_title "
                f"FROM tracks t "
                f"JOIN artists a ON t.artist_id = a.id "
                f"JOIN albums al ON t.album_id = al.id "
                f"WHERE a.name IN ({placeholders}) AND t.file_path IS NOT NULL",
                artist_names
            ).fetchall()
            conn.close()
        else:
            # Scan all library tracks
            with quality_scanner_lock:
                quality_scanner_state["phase"] = "Loading all library tracks..."

            conn = db._get_connection()
            tracks_to_scan = conn.execute(
                "SELECT t.id, t.title, t.artist_id, t.album_id, t.file_path, t.bitrate, a.name as artist_name, al.title as album_title "
                "FROM tracks t "
                "JOIN artists a ON t.artist_id = a.id "
                "JOIN albums al ON t.album_id = al.id "
                "WHERE t.file_path IS NOT NULL"
            ).fetchall()
            conn.close()

        total_tracks = len(tracks_to_scan)
        print(f"📊 [Quality Scanner] Found {total_tracks} tracks to scan")

        with quality_scanner_lock:
            quality_scanner_state["total"] = total_tracks
            quality_scanner_state["phase"] = f"Scanning {total_tracks} tracks..."

        # Initialize Spotify client for matching
        spotify_client = SpotifyClient()
        if not spotify_client.is_authenticated():
            with quality_scanner_lock:
                quality_scanner_state["status"] = "error"
                quality_scanner_state["phase"] = "Spotify not authenticated"
                quality_scanner_state["error_message"] = "Please authenticate with Spotify first"
            print(f"❌ [Quality Scanner] Spotify not authenticated")
            return

        wishlist_service = get_wishlist_service()

        # Scan each track
        for idx, track_row in enumerate(tracks_to_scan, 1):
            try:
                track_id, title, artist_id, album_id, file_path, bitrate, artist_name, album_title = track_row

                # Check quality tier
                tier_name, tier_num = _get_quality_tier_from_extension(file_path)

                # Update progress
                with quality_scanner_lock:
                    quality_scanner_state["processed"] = idx
                    quality_scanner_state["progress"] = (idx / total_tracks) * 100
                    quality_scanner_state["phase"] = f"Scanning: {artist_name} - {title}"

                # Check if meets quality standards
                if tier_num <= min_acceptable_tier:
                    # Quality met
                    with quality_scanner_lock:
                        quality_scanner_state["quality_met"] += 1
                    continue

                # Low quality track found
                with quality_scanner_lock:
                    quality_scanner_state["low_quality"] += 1

                print(f"🔍 [Quality Scanner] Low quality: {artist_name} - {title} ({tier_name}, {file_path})")

                # Attempt to match to Spotify using matching_engine
                matched = False
                matched_track_data = None

                try:
                    # Generate search queries using matching engine
                    temp_track = type('TempTrack', (), {
                        'name': title,
                        'artists': [artist_name],
                        'album': album_title
                    })()

                    search_queries = matching_engine.generate_download_queries(temp_track)
                    print(f"🔍 [Quality Scanner] Generated {len(search_queries)} search queries for {artist_name} - {title}")

                    # Find best match using confidence scoring
                    best_match = None
                    best_confidence = 0.0
                    min_confidence = 0.7  # Match existing standard

                    for query_idx, search_query in enumerate(search_queries):
                        try:
                            spotify_matches = spotify_client.search_tracks(search_query, limit=5)

                            if not spotify_matches:
                                continue

                            # Score each result using matching engine
                            for spotify_track in spotify_matches:
                                try:
                                    # Calculate artist confidence
                                    artist_confidence = 0.0
                                    if spotify_track.artists:
                                        for result_artist in spotify_track.artists:
                                            artist_sim = matching_engine.similarity_score(
                                                matching_engine.normalize_string(artist_name),
                                                matching_engine.normalize_string(result_artist)
                                            )
                                            artist_confidence = max(artist_confidence, artist_sim)

                                    # Calculate title confidence
                                    title_confidence = matching_engine.similarity_score(
                                        matching_engine.normalize_string(title),
                                        matching_engine.normalize_string(spotify_track.name)
                                    )

                                    # Combined confidence (50% artist + 50% title)
                                    combined_confidence = (artist_confidence * 0.5 + title_confidence * 0.5)

                                    print(f"🔍 [Quality Scanner] Candidate: '{spotify_track.artists[0]}' - '{spotify_track.name}' (confidence: {combined_confidence:.3f})")

                                    # Update best match if this is better
                                    if combined_confidence > best_confidence and combined_confidence >= min_confidence:
                                        best_confidence = combined_confidence
                                        best_match = spotify_track
                                        print(f"✅ [Quality Scanner] New best match: {spotify_track.artists[0]} - {spotify_track.name} (confidence: {combined_confidence:.3f})")

                                except Exception as e:
                                    print(f"❌ [Quality Scanner] Error scoring result: {e}")
                                    continue

                            # If we found a very high confidence match, stop searching
                            if best_confidence >= 0.9:
                                print(f"🎯 [Quality Scanner] High confidence match found ({best_confidence:.3f}), stopping search")
                                break

                        except Exception as e:
                            print(f"❌ [Quality Scanner] Error searching with query '{search_query}': {e}")
                            continue

                    # Process best match
                    if best_match:
                        matched = True
                        print(f"✅ [Quality Scanner] Final match: {best_match.artists[0]} - {best_match.name} (confidence: {best_confidence:.3f})")

                        # Build full Spotify track data for wishlist
                        matched_track_data = {
                            'id': best_match.id,
                            'name': best_match.name,
                            'artists': [{'name': artist} for artist in best_match.artists],
                            'album': {
                                'name': best_match.album,
                                'artists': [{'name': artist} for artist in best_match.artists],
                                'album_type': 'album'  # Default to 'album' for quality scanner matches
                            },
                            'duration_ms': best_match.duration_ms,
                            'popularity': best_match.popularity,
                            'preview_url': best_match.preview_url,
                            'external_urls': best_match.external_urls or {}
                        }

                        # Add to wishlist
                        source_context = {
                            'quality_scanner': True,
                            'original_file_path': file_path,
                            'original_format': tier_name,
                            'original_bitrate': bitrate,
                            'match_confidence': best_confidence,
                            'scan_date': datetime.now().isoformat()
                        }

                        success = wishlist_service.add_spotify_track_to_wishlist(
                            spotify_track_data=matched_track_data,
                            failure_reason=f"Low quality - {tier_name.replace('_', ' ').title()} format",
                            source_type='quality_scanner',
                            source_context=source_context,
                            profile_id=profile_id
                        )

                        if success:
                            with quality_scanner_lock:
                                quality_scanner_state["matched"] += 1
                            print(f"✅ [Quality Scanner] Matched and added to wishlist: {artist_name} - {title}")
                        else:
                            print(f"⚠️ [Quality Scanner] Failed to add to wishlist: {artist_name} - {title}")
                    else:
                        print(f"⚠️ [Quality Scanner] No suitable match found (best confidence: {best_confidence:.3f}, required: {min_confidence:.3f})")

                except Exception as matching_error:
                    print(f"❌ [Quality Scanner] Matching error for {artist_name} - {title}: {matching_error}")

                # Store result
                result_entry = {
                    'track_id': track_id,
                    'title': title,
                    'artist': artist_name,
                    'album': album_title,
                    'file_path': file_path,
                    'current_format': tier_name,
                    'bitrate': bitrate,
                    'matched': matched,
                    'spotify_id': matched_track_data['id'] if matched_track_data else None
                }

                with quality_scanner_lock:
                    quality_scanner_state["results"].append(result_entry)

                if not matched:
                    print(f"⚠️ [Quality Scanner] No Spotify match found for: {artist_name} - {title}")

            except Exception as track_error:
                print(f"❌ [Quality Scanner] Error processing track: {track_error}")
                continue

        # Scan complete
        with quality_scanner_lock:
            quality_scanner_state["status"] = "finished"
            quality_scanner_state["progress"] = 100
            quality_scanner_state["phase"] = "Scan complete"

        print(f"✅ [Quality Scanner] Scan complete: {quality_scanner_state['processed']} processed, "
              f"{quality_scanner_state['low_quality']} low quality, {quality_scanner_state['matched']} matched to Spotify")

        # Add activity
        add_activity_item("🔍", "Quality Scan Complete",
                         f"{quality_scanner_state['matched']} tracks added to wishlist", "Now")

        try:
            if automation_engine:
                automation_engine.emit('quality_scan_completed', {
                    'quality_met': str(quality_scanner_state.get('quality_met', 0)),
                    'low_quality': str(quality_scanner_state.get('low_quality', 0)),
                    'total_scanned': str(quality_scanner_state.get('processed', 0)),
                })
        except Exception:
            pass

    except Exception as e:
        print(f"❌ [Quality Scanner] Critical error: {e}")
        import traceback
        traceback.print_exc()

        with quality_scanner_lock:
            quality_scanner_state["status"] = "error"
            quality_scanner_state["error_message"] = str(e)
            quality_scanner_state["phase"] = f"Error: {str(e)}"


def _run_duplicate_cleaner():
    """Main duplicate cleaner worker function - scans Transfer folder for duplicate files"""
    import os
    import shutil
    from collections import defaultdict
    from pathlib import Path

    try:
        with duplicate_cleaner_lock:
            duplicate_cleaner_state["status"] = "running"
            duplicate_cleaner_state["phase"] = "Initializing scan..."
            duplicate_cleaner_state["progress"] = 0
            duplicate_cleaner_state["files_scanned"] = 0
            duplicate_cleaner_state["total_files"] = 0
            duplicate_cleaner_state["duplicates_found"] = 0
            duplicate_cleaner_state["deleted"] = 0
            duplicate_cleaner_state["space_freed"] = 0
            duplicate_cleaner_state["error_message"] = ""

        print(f"🧹 [Duplicate Cleaner] Starting duplicate scan...")

        # Get Transfer folder path from config
        transfer_folder = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer'))
        if not transfer_folder or not os.path.exists(transfer_folder):
            with duplicate_cleaner_lock:
                duplicate_cleaner_state["status"] = "error"
                duplicate_cleaner_state["phase"] = "Transfer folder not configured or does not exist"
                duplicate_cleaner_state["error_message"] = "Please configure Transfer folder in settings"
            print(f"❌ [Duplicate Cleaner] Transfer folder not found: {transfer_folder}")
            return

        # Create deleted folder if it doesn't exist
        deleted_folder = os.path.join(transfer_folder, 'deleted')
        os.makedirs(deleted_folder, exist_ok=True)
        print(f"📁 [Duplicate Cleaner] Deleted folder: {deleted_folder}")

        # Phase 1: Count total files for progress tracking
        with duplicate_cleaner_lock:
            duplicate_cleaner_state["phase"] = "Counting files..."

        total_files = 0
        for root, dirs, files in os.walk(transfer_folder):
            # Skip the deleted folder itself
            if 'deleted' in dirs:
                dirs.remove('deleted')
            total_files += len(files)

        print(f"📊 [Duplicate Cleaner] Found {total_files} total files to scan")

        with duplicate_cleaner_lock:
            duplicate_cleaner_state["total_files"] = total_files
            duplicate_cleaner_state["phase"] = f"Scanning {total_files} files..."

        # Phase 2: Scan and group files by directory and filename
        # Structure: {directory_path: {filename_without_ext: [full_file_paths]}}
        files_by_dir_and_name = defaultdict(lambda: defaultdict(list))
        files_scanned = 0

        # Audio file extensions to consider
        audio_extensions = {'.flac', '.mp3', '.m4a', '.aac', '.opus', '.ogg', '.wav', '.ape', '.wma', '.alac', '.aiff', '.aif', '.dsf', '.dff'}

        for root, dirs, files in os.walk(transfer_folder):
            # Skip the deleted folder
            if 'deleted' in dirs:
                dirs.remove('deleted')

            for file in files:
                files_scanned += 1

                # Update progress
                with duplicate_cleaner_lock:
                    duplicate_cleaner_state["files_scanned"] = files_scanned
                    duplicate_cleaner_state["progress"] = (files_scanned / total_files) * 100 if total_files > 0 else 0
                    duplicate_cleaner_state["phase"] = f"Scanning: {file}"

                # Get file extension
                file_path = os.path.join(root, file)
                file_name, file_ext = os.path.splitext(file)
                file_ext_lower = file_ext.lower()

                # Only process audio files
                if file_ext_lower not in audio_extensions:
                    continue

                # Group by directory and filename (without extension)
                files_by_dir_and_name[root][file_name].append({
                    'full_path': file_path,
                    'extension': file_ext_lower,
                    'size': os.path.getsize(file_path)
                })

        # Phase 3: Process duplicates
        with duplicate_cleaner_lock:
            duplicate_cleaner_state["phase"] = "Processing duplicates..."

        # Quality priority: FLAC > OPUS/OGG > M4A/AAC > MP3/WMA
        format_priority = {
            '.flac': 1, '.ape': 1, '.wav': 1, '.alac': 1, '.aiff': 1, '.aif': 1, '.dsf': 1, '.dff': 1,  # Lossless
            '.opus': 2, '.ogg': 2,  # High quality lossy
            '.m4a': 3, '.aac': 3,   # Standard lossy
            '.mp3': 4, '.wma': 4    # Lower quality lossy
        }

        duplicates_found = 0
        deleted_count = 0
        space_freed = 0

        for directory, files_by_name in files_by_dir_and_name.items():
            for filename, file_versions in files_by_name.items():
                # Only process if we have duplicates (more than one version)
                if len(file_versions) <= 1:
                    continue

                duplicates_found += len(file_versions) - 1  # Count all but the one we keep
                print(f"🔍 [Duplicate Cleaner] Found {len(file_versions)} versions of '{filename}' in {directory}")

                # Sort by priority: best format first, then largest size
                def sort_key(f):
                    priority = format_priority.get(f['extension'], 999)
                    size = f['size']
                    return (priority, -size)  # Negative size for descending order

                sorted_versions = sorted(file_versions, key=sort_key)

                # Keep the first one (best quality), delete the rest
                best_version = sorted_versions[0]
                print(f"✅ [Duplicate Cleaner] Keeping: {os.path.basename(best_version['full_path'])} "
                      f"({best_version['extension']}, {best_version['size']} bytes)")

                for duplicate_file in sorted_versions[1:]:
                    try:
                        # Move to deleted folder with relative path preserved
                        relative_path = os.path.relpath(duplicate_file['full_path'], transfer_folder)
                        deleted_path = os.path.join(deleted_folder, relative_path)

                        # Create subdirectories in deleted folder if needed
                        os.makedirs(os.path.dirname(deleted_path), exist_ok=True)

                        # Move the file
                        shutil.move(duplicate_file['full_path'], deleted_path)

                        # Track stats
                        deleted_count += 1
                        space_freed += duplicate_file['size']

                        print(f"🗑️ [Duplicate Cleaner] Moved to deleted: {os.path.basename(duplicate_file['full_path'])} "
                              f"({duplicate_file['extension']}, {duplicate_file['size']} bytes)")

                        # Update stats
                        with duplicate_cleaner_lock:
                            duplicate_cleaner_state["deleted"] = deleted_count
                            duplicate_cleaner_state["space_freed"] = space_freed
                            duplicate_cleaner_state["duplicates_found"] = duplicates_found

                    except Exception as e:
                        print(f"❌ [Duplicate Cleaner] Error moving file {duplicate_file['full_path']}: {e}")
                        continue

        # Scan complete
        with duplicate_cleaner_lock:
            duplicate_cleaner_state["status"] = "finished"
            duplicate_cleaner_state["progress"] = 100
            duplicate_cleaner_state["phase"] = "Cleaning complete"

        space_mb = space_freed / (1024 * 1024)
        print(f"✅ [Duplicate Cleaner] Scan complete: {files_scanned} files scanned, "
              f"{duplicates_found} duplicates found, {deleted_count} files moved to deleted folder, "
              f"{space_mb:.2f} MB freed")

        # Add activity
        add_activity_item("🧹", "Duplicate Cleaner Complete",
                         f"{deleted_count} files removed, {space_mb:.1f} MB freed", "Now")

        try:
            if automation_engine:
                automation_engine.emit('duplicate_scan_completed', {
                    'files_scanned': str(files_scanned),
                    'duplicates_found': str(duplicates_found),
                    'space_freed': f"{space_mb:.1f} MB",
                })
        except Exception:
            pass

    except Exception as e:
        print(f"❌ [Duplicate Cleaner] Critical error: {e}")
        import traceback
        traceback.print_exc()

        with duplicate_cleaner_lock:
            duplicate_cleaner_state["status"] = "error"
            duplicate_cleaner_state["error_message"] = str(e)
            duplicate_cleaner_state["phase"] = f"Error: {str(e)}"

@app.route('/api/quality-scanner/start', methods=['POST'])
def start_quality_scan():
    """Start the quality scanner"""
    with quality_scanner_lock:
        if quality_scanner_state["status"] == "running":
            return jsonify({"success": False, "error": "A scan is already in progress"}), 409

        data = request.get_json() or {}
        scope = data.get('scope', 'watchlist')  # 'watchlist' or 'all'

        print(f"🔍 [Quality Scanner API] Starting scan with scope: {scope}")

        # Reset state
        quality_scanner_state["status"] = "running"
        quality_scanner_state["phase"] = "Initializing..."
        quality_scanner_state["progress"] = 0
        quality_scanner_state["processed"] = 0
        quality_scanner_state["total"] = 0
        quality_scanner_state["quality_met"] = 0
        quality_scanner_state["low_quality"] = 0
        quality_scanner_state["matched"] = 0
        quality_scanner_state["results"] = []
        quality_scanner_state["error_message"] = ""

        # Submit worker (capture profile_id before thread)
        scan_profile_id = get_current_profile_id()
        quality_scanner_executor.submit(_run_quality_scanner, scope, scan_profile_id)

        add_activity_item("🔍", "Quality Scan Started", f"Scanning {scope} tracks", "Now")

        return jsonify({"success": True, "message": "Quality scan started"})

@app.route('/api/quality-scanner/status', methods=['GET'])
def get_quality_scanner_status():
    """Get current quality scanner status"""
    with quality_scanner_lock:
        return jsonify(quality_scanner_state)

@app.route('/api/quality-scanner/stop', methods=['POST'])
def stop_quality_scan():
    """Stop the quality scanner (sets a stop flag)"""
    with quality_scanner_lock:
        if quality_scanner_state["status"] == "running":
            quality_scanner_state["status"] = "finished"
            quality_scanner_state["phase"] = "Scan stopped by user"
            return jsonify({"success": True, "message": "Stop request sent"})
        else:
            return jsonify({"success": False, "error": "No scan is currently running"}), 404

@app.route('/api/duplicate-cleaner/start', methods=['POST'])
def start_duplicate_cleaner():
    """Start the duplicate cleaner"""
    with duplicate_cleaner_lock:
        if duplicate_cleaner_state["status"] == "running":
            return jsonify({"success": False, "error": "A scan is already in progress"}), 409

        print(f"🧹 [Duplicate Cleaner API] Starting duplicate cleaner...")

        # Reset state
        duplicate_cleaner_state["status"] = "running"
        duplicate_cleaner_state["phase"] = "Initializing..."
        duplicate_cleaner_state["progress"] = 0
        duplicate_cleaner_state["files_scanned"] = 0
        duplicate_cleaner_state["total_files"] = 0
        duplicate_cleaner_state["duplicates_found"] = 0
        duplicate_cleaner_state["deleted"] = 0
        duplicate_cleaner_state["space_freed"] = 0
        duplicate_cleaner_state["error_message"] = ""

        # Submit worker
        duplicate_cleaner_executor.submit(_run_duplicate_cleaner)

        add_activity_item("🧹", "Duplicate Cleaner Started", "Scanning Transfer folder", "Now")

        return jsonify({"success": True, "message": "Duplicate cleaner started"})

@app.route('/api/duplicate-cleaner/status', methods=['GET'])
def get_duplicate_cleaner_status():
    """Get current duplicate cleaner status"""
    with duplicate_cleaner_lock:
        # Convert space_freed from bytes to MB for display
        state_copy = duplicate_cleaner_state.copy()
        state_copy["space_freed_mb"] = duplicate_cleaner_state["space_freed"] / (1024 * 1024)
        return jsonify(state_copy)

@app.route('/api/duplicate-cleaner/stop', methods=['POST'])
def stop_duplicate_cleaner():
    """Stop the duplicate cleaner (sets a stop flag)"""
    with duplicate_cleaner_lock:
        if duplicate_cleaner_state["status"] == "running":
            duplicate_cleaner_state["status"] = "finished"
            duplicate_cleaner_state["phase"] = "Scan stopped by user"
            return jsonify({"success": True, "message": "Stop request sent"})
        else:
            return jsonify({"success": False, "error": "No scan is currently running"}), 404

# ===============================
# == RETAG TOOL ENDPOINTS      ==
# ===============================

@app.route('/api/retag/stats', methods=['GET'])
def get_retag_stats():
    """Get retag tool statistics for the dashboard card."""
    from database.music_database import get_database
    db = get_database()
    stats = db.get_retag_stats()
    return jsonify({"success": True, **stats})

@app.route('/api/retag/groups', methods=['GET'])
def get_retag_groups():
    """Get all retag groups sorted by artist name."""
    from database.music_database import get_database
    db = get_database()
    groups = db.get_retag_groups()
    return jsonify({"success": True, "groups": groups})

@app.route('/api/retag/groups/<int:group_id>/tracks', methods=['GET'])
def get_retag_group_tracks(group_id):
    """Get tracks for a specific retag group."""
    from database.music_database import get_database
    db = get_database()
    tracks = db.get_retag_tracks(group_id)
    return jsonify({"success": True, "tracks": tracks})

@app.route('/api/retag/search', methods=['GET'])
def search_retag_albums():
    """Search for albums to use for retagging (uses Spotify/iTunes fallback)."""
    query = request.args.get('q', '').strip()
    if not query:
        return jsonify({"success": False, "error": "Query parameter 'q' is required"}), 400

    limit = min(int(request.args.get('limit', 12)), 50)
    try:
        results = spotify_client.search_albums(query, limit=limit)
        albums = []
        for a in results:
            albums.append({
                'id': str(a.id),
                'name': a.name,
                'artist': ', '.join(a.artists) if a.artists else 'Unknown Artist',
                'release_date': a.release_date or '',
                'total_tracks': a.total_tracks,
                'image_url': a.image_url,
                'album_type': a.album_type or 'album'
            })
        return jsonify({"success": True, "albums": albums})
    except Exception as e:
        print(f"❌ [Retag] Album search error: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/retag/execute', methods=['POST'])
def execute_retag():
    """Start a retag operation for a group with a new album match."""
    data = request.get_json()
    if not data:
        return jsonify({"success": False, "error": "JSON body required"}), 400

    group_id = data.get('group_id')
    album_id = data.get('album_id')
    if not group_id or not album_id:
        return jsonify({"success": False, "error": "group_id and album_id are required"}), 400

    with retag_lock:
        if retag_state["status"] == "running":
            return jsonify({"success": False, "error": "A retag operation is already running"}), 409

    retag_executor.submit(_execute_retag, group_id, str(album_id))
    return jsonify({"success": True, "message": "Retag operation started"})

@app.route('/api/retag/status', methods=['GET'])
def get_retag_status():
    """Get the current retag operation status."""
    with retag_lock:
        return jsonify(dict(retag_state))

@app.route('/api/retag/groups/<int:group_id>', methods=['DELETE'])
def delete_retag_group(group_id):
    """Delete a retag group (files are NOT deleted)."""
    from database.music_database import get_database
    db = get_database()
    success = db.delete_retag_group(group_id)
    if success:
        return jsonify({"success": True})
    else:
        return jsonify({"success": False, "error": "Group not found"}), 404

@app.route('/api/retag/groups/delete-batch', methods=['POST'])
def delete_retag_groups_batch():
    """Delete multiple retag groups at once."""
    from database.music_database import get_database
    data = request.get_json() or {}
    group_ids = data.get('group_ids', [])
    if not group_ids:
        return jsonify({"success": False, "error": "No group IDs provided"}), 400
    db = get_database()
    removed = 0
    for gid in group_ids:
        if db.delete_retag_group(int(gid)):
            removed += 1
    return jsonify({"success": True, "removed": removed})

@app.route('/api/retag/groups/clear-all', methods=['POST'])
def clear_all_retag_groups():
    """Delete all retag groups."""
    from database.music_database import get_database
    db = get_database()
    count = db.delete_all_retag_groups()
    return jsonify({"success": True, "removed": count})

# ===============================
# == DOWNLOAD MISSING TRACKS   ==
# ===============================

def get_valid_candidates(results, spotify_track, query):
    """
    This function is a direct port from sync.py. It scores and filters
    Soulseek search results against a Spotify track to find the best, most
    accurate download candidates.
    """
    if not results: 
        return []
    # Uses the existing, powerful matching engine for scoring
    initial_candidates = matching_engine.find_best_slskd_matches_enhanced(spotify_track, results)
    if not initial_candidates:
        return []

    # Skip quality filtering for YouTube/Tidal results (quality is fixed by source, not user-selectable per-result)
    is_streaming_source = initial_candidates[0].username in ("youtube", "tidal") if initial_candidates else False

    if is_streaming_source:
        source_label = initial_candidates[0].username.title()
        print(f"🎵 [{source_label}] Skipping quality filter - streaming source handles quality internally")
        quality_filtered_candidates = initial_candidates
    else:
        # Filter by user's quality profile before artist verification (Soulseek only)
        # Use existing soulseek_client to avoid re-initializing (which accesses download_path filesystem)
        quality_filtered_candidates = soulseek_client.soulseek.filter_results_by_quality_preference(initial_candidates)

        # IMPORTANT: Respect empty results from quality filter
        # If user has strict quality requirements (e.g., FLAC-only with fallback disabled),
        # and no results match, we should fail the download rather than force a fallback.
        # The quality filter already has its own fallback logic controlled by the user's settings.
        if not quality_filtered_candidates:
            print(f"⚠️ [Quality Filter] No candidates match quality profile - download will fail per user preferences")
            return []

    verified_candidates = []
    spotify_artists = spotify_track.artists if spotify_track.artists else []

    # Pre-normalize all artist names into word sets using the matching engine
    # This handles Cyrillic, accents, special chars ($), separators, etc.
    artist_word_sets = []
    for artist_name in spotify_artists:
        normalized = matching_engine.normalize_string(artist_name)
        words = set(normalized.split())
        if words:
            artist_word_sets.append(words)

    for candidate in quality_filtered_candidates:
        # Skip artist check for streaming results (title matching is sufficient as processed by matching engine)
        if is_streaming_source:
            verified_candidates.append(candidate)
            continue

        # No artist info available — can't verify, accept candidate
        if not artist_word_sets:
            verified_candidates.append(candidate)
            continue

        # Split the Soulseek path into segments (folders + filename) and check each one.
        # This prevents false positives where a short artist name like "Sia" accidentally
        # matches inside a folder name like "Enthusiastic" — by checking words within
        # individual segments rather than a flat substring of the entire path.
        path_segments = re.split(r'[/\\]', candidate.filename)

        artist_found = False
        for segment in path_segments:
            if not segment:
                continue
            seg_words = set(matching_engine.normalize_string(segment).split())
            if not seg_words:
                continue

            # Check if ANY artist's words are ALL present in this segment
            for artist_words in artist_word_sets:
                if artist_words.issubset(seg_words):
                    artist_found = True
                    break

            if artist_found:
                break

        if artist_found:
            verified_candidates.append(candidate)
    return verified_candidates

def _recover_worker_slot(batch_id, task_id):
    """
    Emergency worker slot recovery function for when normal completion callback fails.
    This prevents permanent worker slot leaks that cause modal to show wrong worker counts.
    """
    try:
        print(f"🚨 [Worker Recovery] Attempting to recover worker slot for batch {batch_id}, task {task_id}")
        
        # Acquire lock with timeout to prevent deadlock
        lock_acquired = tasks_lock.acquire(timeout=3.0)
        if not lock_acquired:
            print(f"💀 [Worker Recovery] FATAL: Could not acquire lock for recovery - worker slot LEAKED")
            return False
            
        try:
            # Verify batch still exists
            if batch_id not in download_batches:
                print(f"⚠️ [Worker Recovery] Batch {batch_id} not found - nothing to recover")
                return True
                
            batch = download_batches[batch_id]
            old_active = batch['active_count']
            
            # Only decrement if there are active workers to prevent negative counts
            if old_active > 0:
                batch['active_count'] -= 1
                new_active = batch['active_count']
                print(f"✅ [Worker Recovery] Recovered worker slot - Active count: {old_active} → {new_active}")
                
                # Try to start next worker if queue isn't empty
                if batch['queue_index'] < len(batch['queue']) and new_active < batch['max_concurrent']:
                    print(f"🔄 [Worker Recovery] Attempting to start replacement worker")
                    # Release lock temporarily to avoid deadlock in _start_next_batch_of_downloads
                    tasks_lock.release()
                    try:
                        _start_next_batch_of_downloads(batch_id)
                    finally:
                        # Re-acquire lock for final cleanup
                        tasks_lock.acquire(timeout=2.0)
                        
                return True
            else:
                print(f"⚠️ [Worker Recovery] Active count already 0 - no recovery needed")
                return True
                
        finally:
            tasks_lock.release()
            
    except Exception as recovery_error:
        print(f"💀 [Worker Recovery] FATAL ERROR in recovery: {recovery_error}")
        return False

def _get_batch_lock(batch_id):
    """Get or create a lock for a specific batch to prevent race conditions"""
    with tasks_lock:
        if batch_id not in batch_locks:
            batch_locks[batch_id] = threading.Lock()
        return batch_locks[batch_id]

def _start_next_batch_of_downloads(batch_id):
    """Start the next batch of downloads up to the concurrent limit (like GUI)"""
    # ENHANCED: Use batch-specific lock to prevent race conditions when multiple threads
    # try to start workers for the same batch concurrently
    batch_lock = _get_batch_lock(batch_id)
    
    with batch_lock:
        # Prevent starting new tasks if shutting down
        if IS_SHUTTING_DOWN:
            print(f"🛑 [Batch Manager] Server shutting down - skipping new tasks for batch {batch_id}")
            return

        with tasks_lock:
            if batch_id not in download_batches:
                return
                
            batch = download_batches[batch_id]
            max_concurrent = batch['max_concurrent']
            queue = batch['queue']
            queue_index = batch['queue_index']
            active_count = batch['active_count']
            
            print(f"🔍 [Batch Lock] Starting workers for {batch_id}: active={active_count}, max={max_concurrent}, queue_pos={queue_index}/{len(queue)}")
            
            # Start downloads up to the concurrent limit
            while active_count < max_concurrent and queue_index < len(queue):
                task_id = queue[queue_index]
                
                # CRITICAL V2 FIX: Skip cancelled tasks instead of trying to restart them
                if task_id in download_tasks:
                    current_status = download_tasks[task_id]['status']
                    if current_status == 'cancelled':
                        print(f"⏭️ [Batch Lock] Skipping cancelled task {task_id} (queue position {queue_index + 1})")
                        download_batches[batch_id]['queue_index'] += 1
                        queue_index += 1
                        continue  # Skip to next task without consuming worker slot
                    
                    # IMPORTANT: Set status to 'searching' BEFORE starting worker (like GUI)
                    # Must be done INSIDE the lock to prevent race conditions with status polling
                    download_tasks[task_id]['status'] = 'searching'
                    download_tasks[task_id]['status_change_time'] = time.time()
                    print(f"🔧 [Batch Manager] Set task {task_id} status to 'searching'")
                else:
                    print(f"⚠️ [Batch Lock] Task {task_id} not found in download_tasks - skipping")
                    download_batches[batch_id]['queue_index'] += 1
                    queue_index += 1
                    continue
                
                # CRITICAL FIX: Submit to executor BEFORE incrementing counters to prevent ghost workers
                try:
                    # Submit to executor first - this can fail
                    future = missing_download_executor.submit(_download_track_worker, task_id, batch_id)
                    
                    # Only increment counters AFTER successful submit
                    download_batches[batch_id]['active_count'] += 1
                    download_batches[batch_id]['queue_index'] += 1
                    
                    print(f"🔄 [Batch Lock] Started download {queue_index + 1}/{len(queue)} - Active: {active_count + 1}/{max_concurrent}")
                    
                    # Update local counters for next iteration
                    active_count += 1
                    queue_index += 1
                    
                except Exception as submit_error:
                    print(f"❌ [Batch Lock] CRITICAL: Failed to submit task {task_id} to executor: {submit_error}")
                    print(f"🚨 [Batch Lock] Worker slot NOT consumed - preventing ghost worker")
                    
                    # Reset task status since worker never started
                    if task_id in download_tasks:
                        download_tasks[task_id]['status'] = 'failed'
                        print(f"🔧 [Batch Lock] Set task {task_id} status to 'failed' due to submit failure")
                    
                    # Don't increment counters - no worker was actually started
                    # This prevents the "ghost worker" issue where active_count is incremented but no actual worker runs
                    break  # Stop trying to start more workers if executor is failing
            
            print(f"✅ [Batch Lock] Finished starting workers for {batch_id}: final_active={download_batches[batch_id]['active_count']}, max={max_concurrent}")

def _get_track_artist_name(track_info):
    """Extract artist name from track info, handling different data formats (replicating sync.py)"""
    if not track_info:
        return "Unknown Artist"
    
    # Handle Spotify API format with artists array
    artists = track_info.get('artists', [])
    if artists and len(artists) > 0:
        if isinstance(artists[0], dict) and 'name' in artists[0]:
            return artists[0]['name']
        elif isinstance(artists[0], str):
            return artists[0]
    
    # Fallback to single artist field
    artist = track_info.get('artist')
    if artist:
        return artist
        
    return "Unknown Artist"

def _ensure_spotify_track_format(track_info):
    """
    Ensure track_info has proper Spotify track structure for wishlist service.
    Converts webui track format to match sync.py's spotify_track format.
    """
    if not track_info:
        return {}
    
    # If it already has the proper Spotify structure, return as-is
    if isinstance(track_info.get('artists'), list) and len(track_info.get('artists', [])) > 0:
        first_artist = track_info['artists'][0]
        if isinstance(first_artist, dict) and 'name' in first_artist:
            # Already has proper Spotify format
            return track_info
    
    # Convert to proper Spotify format
    artists_list = []
    
    # Handle different artist formats from webui
    artists = track_info.get('artists', [])
    if artists:
        if isinstance(artists, list):
            for artist in artists:
                if isinstance(artist, dict) and 'name' in artist:
                    artists_list.append({'name': artist['name']})
                elif isinstance(artist, str):
                    artists_list.append({'name': artist})
                else:
                    artists_list.append({'name': str(artist)})
        else:
            # Single artist as string
            artists_list.append({'name': str(artists)})
    else:
        # Fallback: try single artist field
        artist = track_info.get('artist')
        if artist:
            artists_list.append({'name': str(artist)})
        else:
            artists_list.append({'name': 'Unknown Artist'})
    
    # Build album object — preserve ALL fields (id, release_date, total_tracks,
    # album_type, images, etc.) so wishlist tracks retain full album context
    # for correct folder placement, multi-disc support, and classification
    album_data = track_info.get('album', {})
    if isinstance(album_data, dict):
        album = dict(album_data)  # Copy all fields
        album.setdefault('name', 'Unknown Album')
    else:
        album = {
            'name': str(album_data) if album_data else 'Unknown Album'
        }

    # Build proper Spotify track structure
    spotify_track = {
        'id': track_info.get('id', f"webui_{hash(str(track_info))}"),
        'name': track_info.get('name', 'Unknown Track'),
        'artists': artists_list,  # Proper Spotify format
        'album': album,
        'duration_ms': track_info.get('duration_ms', 0),
        'track_number': track_info.get('track_number', 1),
        'disc_number': track_info.get('disc_number', 1),
        'preview_url': track_info.get('preview_url'),
        'external_urls': track_info.get('external_urls', {}),
        'popularity': track_info.get('popularity', 0),
        'source': 'webui_modal'  # Mark as coming from webui
    }

    return spotify_track

def _process_failed_tracks_to_wishlist_exact(batch_id):
    """
    Process failed and cancelled tracks to wishlist - EXACT replication of sync.py's on_all_downloads_complete() logic.
    This matches sync.py's behavior precisely.
    """
    try:
        from core.wishlist_service import get_wishlist_service
        from datetime import datetime
        
        print(f"🔍 [Wishlist Processing] Starting wishlist processing for batch {batch_id}")
        
        with tasks_lock:
            if batch_id not in download_batches:
                print(f"⚠️ [Wishlist Processing] Batch {batch_id} not found")
                return {'tracks_added': 0, 'errors': 0}
        
        batch = download_batches[batch_id]
        permanently_failed_tracks = batch.get('permanently_failed_tracks', [])
        cancelled_tracks = batch.get('cancelled_tracks', set())
        
        # STEP 0: Remove completed tracks from wishlist (THIS WAS MISSING!)
        print(f"🔍 [Wishlist Processing] Checking completed tracks for wishlist removal")
        for task_id in batch.get('queue', []):
            if task_id in download_tasks:
                task = download_tasks[task_id]
                if task.get('status') == 'completed':
                    try:
                        track_info = task.get('track_info', {})
                        context = {'track_info': track_info, 'original_search_result': track_info}
                        _check_and_remove_from_wishlist(context)
                    except Exception as e:
                        print(f"⚠️ [Wishlist Processing] Error removing completed track from wishlist: {e}")
        
        # STEP 1: Add cancelled tracks that were missing to permanently_failed_tracks (replicating sync.py)
        # This matches sync.py's logic for adding cancelled missing tracks to the failed list
        if cancelled_tracks:
            print(f"🔍 [Wishlist Processing] Processing {len(cancelled_tracks)} cancelled tracks")
            
            # Process cancelled tracks with safeguard to prevent infinite loops
            processed_count = 0
            max_process = 100  # Safety limit
            
            with tasks_lock:
                for task_id in batch.get('queue', [])[:max_process]:  # Limit processing
                    if task_id in download_tasks:
                        task = download_tasks[task_id]
                        track_index = task.get('track_index', 0)
                        if track_index in cancelled_tracks:
                            # Check if track was actually missing (not successfully downloaded)
                            task_status = task.get('status', 'unknown')
                            if task_status != 'completed':
                                # Build cancelled track info matching sync.py format
                                original_track_info = task.get('track_info', {})
                                spotify_track_data = _ensure_spotify_track_format(original_track_info)
                                
                                cancelled_track_info = {
                                    'download_index': track_index,
                                    'table_index': track_index,
                                    'track_name': original_track_info.get('name', 'Unknown Track'),
                                    'artist_name': _get_track_artist_name(original_track_info),
                                    'retry_count': 0,
                                    'spotify_track': spotify_track_data,  # Properly formatted spotify track
                                    'failure_reason': 'Download cancelled',
                                    'candidates': task.get('cached_candidates', [])
                                }
                                
                                # Check if not already in permanently_failed_tracks (sync.py does this check)
                                if not any(t.get('table_index') == track_index for t in permanently_failed_tracks):
                                    permanently_failed_tracks.append(cancelled_track_info)
                                    processed_count += 1
                                    print(f"🚫 [Wishlist Processing] Added cancelled missing track {cancelled_track_info['track_name']} to failed list for wishlist")
            
            print(f"🔍 [Wishlist Processing] Processed {processed_count} cancelled tracks")

        # STEP 1.5: Recover any failed/not_found tasks not captured in permanently_failed_tracks.
        # Stuck detection (in _on_download_completed, _check_batch_completion_v2, and the Safety Valve)
        # can force-mark tasks as not_found/failed without adding them to permanently_failed_tracks,
        # causing them to silently skip wishlist processing.
        with tasks_lock:
            for task_id in batch.get('queue', []):
                if task_id in download_tasks:
                    task = download_tasks[task_id]
                    if task['status'] in ('failed', 'not_found'):
                        track_index = task.get('track_index', 0)
                        if not any(t.get('table_index') == track_index for t in permanently_failed_tracks):
                            original_track_info = task.get('track_info', {})
                            spotify_track_data = _ensure_spotify_track_format(original_track_info)
                            recovered_track_info = {
                                'download_index': track_index,
                                'table_index': track_index,
                                'track_name': original_track_info.get('name', 'Unknown Track'),
                                'artist_name': _get_track_artist_name(original_track_info),
                                'retry_count': task.get('retry_count', 0),
                                'spotify_track': spotify_track_data,
                                'failure_reason': task.get('error_message', 'Download failed'),
                                'candidates': task.get('cached_candidates', [])
                            }
                            permanently_failed_tracks.append(recovered_track_info)
                            print(f"📋 [Wishlist Processing] Recovered uncaptured failed track for wishlist: {recovered_track_info['track_name']}")

        # STEP 2: Add permanently failed tracks to wishlist (exact sync.py logic)
        failed_count = len(permanently_failed_tracks)
        wishlist_added_count = 0
        error_count = 0
        
        print(f"🔍 [Wishlist Processing] Processing {failed_count} failed tracks for wishlist")
        
        if permanently_failed_tracks:
            try:
                wishlist_service = get_wishlist_service()
                
                # Create source_context identical to sync.py
                source_context = {
                    'playlist_name': batch.get('playlist_name', 'Unknown Playlist'),
                    'playlist_id': batch.get('playlist_id', None),
                    'added_from': 'webui_modal',  # Distinguish from sync_page_modal
                    'timestamp': datetime.now().isoformat()
                }
                
                # Process each failed track (matching sync.py's loop) with safety limit
                max_failed_tracks = min(len(permanently_failed_tracks), 50)  # Safety limit
                for i, failed_track_info in enumerate(permanently_failed_tracks[:max_failed_tracks]):
                    try:
                        track_name = failed_track_info.get('track_name', f'Track {i+1}')
                        print(f"🔍 [Wishlist Processing] Adding track {i+1}/{max_failed_tracks}: {track_name}")
                        
                        success = wishlist_service.add_failed_track_from_modal(
                            track_info=failed_track_info,
                            source_type='playlist',
                            source_context=source_context,
                            profile_id=batch.get('profile_id', 1)
                        )
                        if success:
                            wishlist_added_count += 1
                            print(f"✅ [Wishlist Processing] Added {track_name} to wishlist")
                            try:
                                if automation_engine:
                                    automation_engine.emit('wishlist_item_added', {
                                        'artist': artist_name,
                                        'title': track_name,
                                        'reason': track.get('failure_reason', ''),
                                    })
                            except Exception:
                                pass
                        else:
                            print(f"⚠️ [Wishlist Processing] Failed to add {track_name} to wishlist")
                            
                    except Exception as e:
                        error_count += 1
                        print(f"❌ [Wishlist Processing] Exception adding track to wishlist: {e}")
                
                print(f"✨ [Wishlist Processing] Added {wishlist_added_count}/{failed_count} failed tracks to wishlist (errors: {error_count})")
                        
            except Exception as e:
                error_count = len(permanently_failed_tracks)
                print(f"❌ [Wishlist Processing] Critical error adding failed tracks to wishlist: {e}")
                import traceback
                traceback.print_exc()
        else:
            print(f"ℹ️ [Wishlist Processing] No failed tracks to add to wishlist")
        
        # Store completion summary in batch for API response (matching sync.py pattern)
        completion_summary = {
            'tracks_added': wishlist_added_count,
            'errors': error_count,
            'total_failed': failed_count
        }
        
        with tasks_lock:
            if batch_id in download_batches:
                download_batches[batch_id]['wishlist_summary'] = completion_summary
                download_batches[batch_id]['wishlist_processing_complete'] = True
                # Phase already set to 'complete' in _on_download_completed

        print(f"✅ [Wishlist Processing] Completed wishlist processing for batch {batch_id}")

        # Auto-cleanup: Clear completed downloads from slskd
        try:
            logger.info(f"🧹 [Auto-Cleanup] Clearing completed downloads from slskd after batch {batch_id}")
            run_async(soulseek_client.clear_all_completed_downloads())
            logger.info(f"✅ [Auto-Cleanup] Completed downloads cleared from slskd")
        except Exception as cleanup_error:
            logger.warning(f"⚠️ [Auto-Cleanup] Failed to clear completed downloads: {cleanup_error}")

        # Sweep empty directories left behind by this batch's downloads
        try:
            _sweep_empty_download_directories()
        except Exception as sweep_error:
            logger.warning(f"⚠️ [Auto-Cleanup] Failed to sweep empty directories: {sweep_error}")

        return completion_summary
    
    except Exception as e:
        print(f"❌ [Wishlist Processing] CRITICAL ERROR in wishlist processing: {e}")
        import traceback
        traceback.print_exc()
        
        # Mark batch as complete even with errors to prevent infinite loops
        try:
            with tasks_lock:
                if batch_id in download_batches:
                    download_batches[batch_id]['phase'] = 'complete'
                    download_batches[batch_id]['completion_time'] = time.time()  # Track for auto-cleanup
                    download_batches[batch_id]['wishlist_summary'] = {
                        'tracks_added': 0,
                        'errors': 1,
                        'total_failed': 0,
                        'error_message': str(e)
                    }
                    download_batches[batch_id]['wishlist_processing_complete'] = True
        except Exception as lock_error:
            print(f"❌ [Wishlist Processing] Failed to update batch after error: {lock_error}")
        
        return {'tracks_added': 0, 'errors': 1, 'total_failed': 0}

def _process_failed_tracks_to_wishlist_exact_with_auto_completion(batch_id):
    """
    Process failed tracks to wishlist for auto-initiated batches and handle auto-processing completion.
    This extends the standard processing with automatic scheduling of the next cycle.
    """
    global wishlist_auto_processing
    
    try:
        print(f"🤖 [Auto-Wishlist] Processing completion for auto-initiated batch {batch_id}")
        
        # Run standard wishlist processing
        completion_summary = _process_failed_tracks_to_wishlist_exact(batch_id)
        
        # Log auto-processing completion
        tracks_added = completion_summary.get('tracks_added', 0)
        total_failed = completion_summary.get('total_failed', 0)
        print(f"🎉 [Auto-Wishlist] Background processing complete: {tracks_added} added to wishlist, {total_failed} failed")
        
        # Add activity for wishlist processing
        if tracks_added > 0:
            add_activity_item("⭐", "Wishlist Updated", f"{tracks_added} failed tracks added to wishlist", "Now")

        # TOGGLE CYCLE: Switch to next category for next run
        try:
            with tasks_lock:
                if batch_id in download_batches:
                    current_cycle = download_batches[batch_id].get('current_cycle', 'albums')
                else:
                    current_cycle = 'albums'  # Default fallback

            next_cycle = 'singles' if current_cycle == 'albums' else 'albums'

            from database.music_database import MusicDatabase
            db = MusicDatabase()
            with db._get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute("""
                    INSERT OR REPLACE INTO metadata (key, value, updated_at)
                    VALUES ('wishlist_cycle', ?, CURRENT_TIMESTAMP)
                """, (next_cycle,))
                conn.commit()

            print(f"🔄 [Auto-Wishlist] Cycle toggled after completion: {current_cycle} → {next_cycle}")
        except Exception as cycle_error:
            print(f"⚠️ [Auto-Wishlist] Error toggling cycle: {cycle_error}")

        # Mark auto-processing as complete and reset timestamp
        with wishlist_timer_lock:
            wishlist_auto_processing = False
            wishlist_auto_processing_timestamp = 0

        try:
            if automation_engine:
                automation_engine.emit('wishlist_processing_completed', {
                    'tracks_processed': str(total_failed),
                    'tracks_found': str(tracks_added),
                    'tracks_failed': str(total_failed - tracks_added),
                })
        except Exception:
            pass

        return completion_summary

    except Exception as e:
        print(f"❌ [Auto-Wishlist] Error in auto-completion processing: {e}")
        import traceback
        traceback.print_exc()

        # Ensure auto-processing flag is reset even on error and reset timestamp
        with wishlist_timer_lock:
            wishlist_auto_processing = False
            wishlist_auto_processing_timestamp = 0

        return {'tracks_added': 0, 'errors': 1, 'total_failed': 0}

def _on_download_completed(batch_id, task_id, success=True):
    """Called when a download completes to start the next one in queue"""
    with tasks_lock:
        if batch_id not in download_batches:
            print(f"⚠️ [Batch Manager] Batch {batch_id} not found for completed task {task_id}")
            return

        # Guard against double-calling: track which tasks have already been completed
        # This prevents active_count from being decremented multiple times for the same task
        # (e.g. monitor detects completion AND post-processing calls this again)
        completed_tasks = download_batches[batch_id].setdefault('_completed_task_ids', set())
        if task_id in completed_tasks:
            print(f"⚠️ [Batch Manager] Task {task_id} already completed — skipping duplicate _on_download_completed call")
            # Set terminal status so the monitor loop stops re-processing this task
            if task_id in download_tasks and download_tasks[task_id].get('status') in ('downloading', 'queued'):
                download_tasks[task_id]['status'] = 'completed'
            return
        completed_tasks.add(task_id)

        # Track failed/cancelled tasks in batch state (replicating sync.py)
        if not success and task_id in download_tasks:
            task = download_tasks[task_id]
            task_status = task.get('status', 'unknown')
            
            # Build track_info structure matching sync.py's permanently_failed_tracks format
            original_track_info = task.get('track_info', {})
            
            # Ensure spotify_track has proper structure for wishlist service
            spotify_track_data = _ensure_spotify_track_format(original_track_info)
            
            track_info = {
                'download_index': task.get('track_index', 0),
                'table_index': task.get('track_index', 0), 
                'track_name': original_track_info.get('name', 'Unknown Track'),
                'artist_name': _get_track_artist_name(original_track_info),
                'retry_count': task.get('retry_count', 0),
                'spotify_track': spotify_track_data,  # Properly formatted spotify track for wishlist
                'failure_reason': 'Download cancelled' if task_status == 'cancelled' else ('Not found on Soulseek' if task_status == 'not_found' else 'Download failed'),
                'candidates': task.get('cached_candidates', [])  # Include search results if available
            }
            
            if task_status == 'cancelled':
                download_batches[batch_id]['cancelled_tracks'].add(task.get('track_index', 0))
                print(f"🚫 [Batch Manager] Added cancelled track to batch tracking: {track_info['track_name']}")
                add_activity_item("🚫", "Download Cancelled", f"'{track_info['track_name']}'", "Now")
            elif task_status in ('failed', 'not_found'):
                download_batches[batch_id]['permanently_failed_tracks'].append(track_info)
                if task_status == 'not_found':
                    print(f"🔇 [Batch Manager] Added not-found track to batch tracking: {track_info['track_name']}")
                    add_activity_item("🔇", "Not Found", f"'{track_info['track_name']}'", "Now")
                else:
                    print(f"❌ [Batch Manager] Added failed track to batch tracking: {track_info['track_name']}")
                    add_activity_item("❌", "Download Failed", f"'{track_info['track_name']}'", "Now")

                try:
                    if automation_engine:
                        automation_engine.emit('download_failed', {
                            'artist': track_info.get('artist_name', ''),
                            'title': track_info.get('track_name', ''),
                            'reason': track_info.get('failure_reason', 'Unknown'),
                        })
                except Exception:
                    pass

        # WISHLIST REMOVAL: Handle successful downloads for wishlist removal
        if success and task_id in download_tasks:
            try:
                task = download_tasks[task_id]
                track_info = task.get('track_info', {})
                print(f"📋 [Batch Manager] Successful download - checking wishlist removal for task {task_id}")
                
                # Add activity for successful download
                track_name = track_info.get('name', 'Unknown Track')

                # Safely extract artist name (handle both list and string formats)
                artists = track_info.get('artists', [])
                if isinstance(artists, list) and len(artists) > 0:
                    first_artist = artists[0]
                    artist_name = first_artist.get('name', 'Unknown Artist') if isinstance(first_artist, dict) else str(first_artist)
                elif isinstance(artists, str):
                    artist_name = artists
                else:
                    artist_name = 'Unknown Artist'

                add_activity_item("📥", "Download Complete", f"'{track_name}' by {artist_name}", "Now")
                
                # Try to remove from wishlist using track info
                if track_info:
                    # Create a context-like structure for the wishlist removal function
                    context = {
                        'track_info': track_info,
                        'original_search_result': track_info  # fallback
                    }
                    _check_and_remove_from_wishlist(context)
            except Exception as wishlist_error:
                print(f"⚠️ [Batch Manager] Error checking wishlist removal for successful download: {wishlist_error}")
        
        # Decrement active count
        old_active = download_batches[batch_id]['active_count']
        download_batches[batch_id]['active_count'] -= 1
        new_active = download_batches[batch_id]['active_count']
        
        print(f"🔄 [Batch Manager] Task {task_id} completed ({'success' if success else 'failed/cancelled'}). Active workers: {old_active} → {new_active}/{download_batches[batch_id]['max_concurrent']}")
        
        # ENHANCED: Always check batch completion after any task completes
        # This ensures completion is detected even when mixing normal downloads with cancelled tasks
        print(f"🔍 [Batch Manager] Checking batch completion after task {task_id} completed")
        
        # FIXED: Check if batch is truly complete (all tasks finished, not just workers freed)
        batch = download_batches[batch_id]
        all_tasks_started = batch['queue_index'] >= len(batch['queue'])
        no_active_workers = batch['active_count'] == 0
        
        # Count actually finished tasks (completed, failed, or cancelled)
        # CRITICAL: Don't include 'post_processing' as finished - it's still in progress (unless stuck)!
        # CRITICAL: Don't include 'searching' as finished - task is being retried (unless stuck)!
        finished_count = 0
        retrying_count = 0
        queue = batch.get('queue', [])
        current_time = time.time()
        for task_id in queue:
            if task_id in download_tasks:
                task = download_tasks[task_id]
                task_status = task['status']

                # STUCK DETECTION: Force fail tasks that have been in transitional states too long
                if task_status == 'searching':
                    task_age = current_time - task.get('status_change_time', current_time)
                    if task_age > 600:  # 10 minutes
                        print(f"⏰ [Stuck Detection] Task {task_id} stuck in searching for {task_age:.0f}s - forcing not_found")
                        task['status'] = 'not_found'
                        task['error_message'] = f'Search stuck for {int(task_age // 60)} minutes with no results — timed out'
                        finished_count += 1
                    else:
                        retrying_count += 1
                elif task_status == 'post_processing':
                    task_age = current_time - task.get('status_change_time', current_time)
                    if task_age > 300:  # 5 minutes (post-processing should be fast)
                        print(f"⏰ [Stuck Detection] Task {task_id} stuck in post_processing for {task_age:.0f}s - forcing completion")
                        task['status'] = 'completed'  # Assume it worked if file verification is taking too long
                        finished_count += 1
                    else:
                        retrying_count += 1
                elif task_status in ['completed', 'failed', 'cancelled', 'not_found']:
                    finished_count += 1
            else:
                # Task ID in queue but not in download_tasks - treat as completed to prevent blocking
                print(f"⚠️ [Orphaned Task] Task {task_id} in queue but not in download_tasks - counting as finished")
                finished_count += 1
        
        all_tasks_truly_finished = finished_count >= len(queue)
        has_retrying_tasks = retrying_count > 0
        
        if all_tasks_started and no_active_workers and all_tasks_truly_finished and not has_retrying_tasks:
            print(f"🎉 [Batch Manager] Batch {batch_id} truly complete - all {finished_count}/{len(queue)} tasks finished - processing failed tracks to wishlist")
        elif all_tasks_started and no_active_workers and has_retrying_tasks:
            print(f"🔄 [Batch Manager] Batch {batch_id}: all workers free but {retrying_count} tasks retrying - continuing monitoring")
        elif all_tasks_started and no_active_workers:
            # This used to incorrectly mark batch as complete!
            print(f"📊 [Batch Manager] Batch {batch_id}: all workers free but only {finished_count}/{len(queue)} tasks finished - continuing monitoring")
        
        if all_tasks_started and no_active_workers and all_tasks_truly_finished and not has_retrying_tasks:
            
            # Check if this is an auto-initiated batch
            is_auto_batch = batch.get('auto_initiated', False)
            
            # FIXED: Ensure batch is not already marked as complete to prevent duplicate processing
            if batch.get('phase') != 'complete':
                # Mark batch as complete and set completion timestamp for auto-cleanup
                batch['phase'] = 'complete'
                batch['completion_time'] = time.time()  # Track when batch completed
            
            # Add activity for batch completion
            playlist_name = batch.get('playlist_name', 'Unknown Playlist')
            failed_count = len(batch.get('permanently_failed_tracks', []))
            successful_downloads = finished_count - failed_count
            add_activity_item("✅", "Download Batch Complete", f"'{playlist_name}' - {successful_downloads} tracks downloaded", "Now")

            # Emit batch_complete event for automation engine
            try:
                if automation_engine:
                    automation_engine.emit('batch_complete', {
                        'playlist_name': playlist_name,
                        'total_tracks': str(len(queue)),
                        'completed_tracks': str(successful_downloads),
                        'failed_tracks': str(failed_count),
                    })
            except Exception:
                pass
            
            # Update YouTube playlist phase to 'download_complete' if this is a YouTube playlist
            playlist_id = batch.get('playlist_id')
            if playlist_id and playlist_id.startswith('youtube_'):
                url_hash = playlist_id.replace('youtube_', '')
                if url_hash in youtube_playlist_states:
                    youtube_playlist_states[url_hash]['phase'] = 'download_complete'
                    print(f"📋 Updated YouTube playlist {url_hash} to download_complete phase")
            
            # Update Tidal playlist phase to 'download_complete' if this is a Tidal playlist
            if playlist_id and playlist_id.startswith('tidal_'):
                tidal_playlist_id = playlist_id.replace('tidal_', '')
                if tidal_playlist_id in tidal_discovery_states:
                    tidal_discovery_states[tidal_playlist_id]['phase'] = 'download_complete'
                    print(f"📋 Updated Tidal playlist {tidal_playlist_id} to download_complete phase")
            
            print(f"🎉 [Batch Manager] Batch {batch_id} complete - stopping monitor")
            download_monitor.stop_monitoring(batch_id)

            # REPAIR: Scan all album folders from this batch for track number issues
            if repair_worker:
                repair_worker.process_batch(batch_id)

            # Mark that wishlist processing is starting (prevents premature cleanup)
            batch['wishlist_processing_started'] = True

            # Process wishlist outside of the lock to prevent threading issues
            if is_auto_batch:
                # For auto-initiated batches, handle completion and schedule next cycle
                missing_download_executor.submit(_process_failed_tracks_to_wishlist_exact_with_auto_completion, batch_id)
            else:
                # For manual batches, use standard wishlist processing
                missing_download_executor.submit(_process_failed_tracks_to_wishlist_exact, batch_id)
            return  # Don't start next batch if we're done
    
    # Start next downloads in queue
    print(f"🔄 [Batch Manager] Starting next batch for {batch_id}")
    _start_next_batch_of_downloads(batch_id)

def _run_full_missing_tracks_process(batch_id, playlist_id, tracks_json):
    """
    A master worker that handles the entire missing tracks process:
    1. Runs the analysis.
    2. If missing tracks are found, it automatically queues them for download.
    """
    try:
        # PHASE 1: ANALYSIS
        with tasks_lock:
            if batch_id in download_batches:
                download_batches[batch_id]['phase'] = 'analysis'
                download_batches[batch_id]['analysis_total'] = len(tracks_json)
                download_batches[batch_id]['analysis_processed'] = 0

        from database.music_database import MusicDatabase
        db = MusicDatabase()
        active_server = config_manager.get_active_media_server()
        analysis_results = []

        # Get force download flag from batch
        force_download_all = False
        with tasks_lock:
            if batch_id in download_batches:
                force_download_all = download_batches[batch_id].get('force_download_all', False)

        if force_download_all:
            print(f"🔄 [Force Download] Force download mode enabled for batch {batch_id} - treating all tracks as missing")

        for i, track_data in enumerate(tracks_json):
            # Use original table index if provided (for partial track selection),
            # otherwise fall back to enumeration index
            track_index = track_data.get('_original_index', i)
            track_name = track_data.get('name', '')
            artists = track_data.get('artists', [])
            found, confidence = False, 0.0

            # Skip database check if force download is enabled
            if force_download_all:
                print(f"🔄 [Force Download] Skipping database check for '{track_name}' - treating as missing")
                found, confidence = False, 0.0
            else:
                for artist in artists:
                    # Handle both string format and Spotify API format {'name': 'Artist Name'}
                    if isinstance(artist, str):
                        artist_name = artist
                    elif isinstance(artist, dict) and 'name' in artist:
                        artist_name = artist['name']
                    else:
                        artist_name = str(artist)
                    db_track, track_confidence = db.check_track_exists(
                        track_name, artist_name, confidence_threshold=0.7, server_source=active_server
                    )
                    if db_track and track_confidence >= 0.7:
                        found, confidence = True, track_confidence
                        break

            analysis_results.append({
                'track_index': track_index, 'track': track_data, 'found': found, 'confidence': confidence
            })
            
            # WISHLIST REMOVAL: If track is found in database, check if it should be removed from wishlist
            if found and confidence >= 0.7:
                try:
                    _check_and_remove_track_from_wishlist_by_metadata(track_data)
                except Exception as wishlist_error:
                    print(f"⚠️ [Analysis] Error checking wishlist removal for found track: {wishlist_error}")

            with tasks_lock:
                if batch_id in download_batches:
                    download_batches[batch_id]['analysis_processed'] = i + 1
                    # Store incremental results for live updates
                    download_batches[batch_id]['analysis_results'] = analysis_results.copy()

        missing_tracks = [res for res in analysis_results if not res['found']]

        with tasks_lock:
            if batch_id in download_batches:
                download_batches[batch_id]['analysis_results'] = analysis_results

        # PHASE 2: TRANSITION TO DOWNLOAD (if necessary)
        if not missing_tracks:
            print(f"✅ Analysis for batch {batch_id} complete. No missing tracks.")

            is_auto_batch = False
            with tasks_lock:
                if batch_id in download_batches:
                    is_auto_batch = download_batches[batch_id].get('auto_initiated', False)
                    download_batches[batch_id]['phase'] = 'complete'
                    download_batches[batch_id]['completion_time'] = time.time()  # Track for auto-cleanup

                    # Update YouTube playlist phase to 'download_complete' if this is a YouTube playlist
                    if playlist_id.startswith('youtube_'):
                        url_hash = playlist_id.replace('youtube_', '')
                        if url_hash in youtube_playlist_states:
                            youtube_playlist_states[url_hash]['phase'] = 'download_complete'
                            print(f"📋 Updated YouTube playlist {url_hash} to download_complete phase (no missing tracks)")

                    # Update Tidal playlist phase to 'download_complete' if this is a Tidal playlist
                    if playlist_id.startswith('tidal_'):
                        tidal_playlist_id = playlist_id.replace('tidal_', '')
                        if tidal_playlist_id in tidal_discovery_states:
                            tidal_discovery_states[tidal_playlist_id]['phase'] = 'download_complete'
                            print(f"📋 Updated Tidal playlist {tidal_playlist_id} to download_complete phase (no missing tracks)")

            # Handle auto-initiated wishlist completion even when no missing tracks
            if is_auto_batch and playlist_id == 'wishlist':
                print("🤖 [Auto-Wishlist] No missing tracks found - calling auto-completion handler to toggle cycle and reschedule")
                missing_download_executor.submit(_process_failed_tracks_to_wishlist_exact_with_auto_completion, batch_id)

            return

        print(f" transitioning batch {batch_id} to download phase with {len(missing_tracks)} tracks.")

        with tasks_lock:
            if batch_id not in download_batches: return

            download_batches[batch_id]['phase'] = 'downloading'

            # Get batch album context (if this is an artist album download)
            batch = download_batches[batch_id]
            batch_album_context = batch.get('album_context')
            batch_artist_context = batch.get('artist_context')
            batch_is_album = batch.get('is_album_download', False)
            batch_playlist_folder_mode = batch.get('playlist_folder_mode', False)
            batch_playlist_name = batch.get('playlist_name', 'Unknown Playlist')

            # Compute total_discs for multi-disc album subfolder support
            # Use ALL tracks (tracks_json), not just missing ones, to correctly detect multi-disc
            # even when only one disc has missing tracks
            if batch_is_album and batch_album_context:
                total_discs = max((t.get('disc_number', 1) for t in tracks_json), default=1)
                batch_album_context['total_discs'] = total_discs
                if total_discs > 1:
                    print(f"💿 [Multi-Disc] Detected {total_discs} discs for album '{batch_album_context.get('name')}'")

            # Pre-compute total_discs per album for wishlist tracks (grouped by album ID)
            # Wishlist tracks aren't batch_is_album but each track has disc_number in spotify_data
            wishlist_album_disc_counts = {}
            if playlist_id == 'wishlist':
                import json as _json
                # First pass: collect disc_number from stored spotify_data
                for t in tracks_json:
                    sp_data = t.get('spotify_data', {})
                    if isinstance(sp_data, str):
                        try:
                            sp_data = _json.loads(sp_data)
                        except:
                            sp_data = {}
                    album_id = (sp_data.get('album', {}) or {}).get('id')
                    disc_num = sp_data.get('disc_number', t.get('disc_number', 1))
                    if album_id:
                        wishlist_album_disc_counts[album_id] = max(
                            wishlist_album_disc_counts.get(album_id, 1), disc_num
                        )


            for res in missing_tracks:
                task_id = str(uuid.uuid4())
                track_info = res['track'].copy()

                # Add explicit album context to track_info for artist album downloads
                if batch_is_album and batch_album_context and batch_artist_context:
                    track_info['_explicit_album_context'] = batch_album_context
                    track_info['_explicit_artist_context'] = batch_artist_context
                    track_info['_is_explicit_album_download'] = True
                    print(f"🎵 [Task Creation] Added explicit album context for: {track_info.get('name')}")

                # SPECIAL WISHLIST HANDLING: Inject album context if available to force grouping
                elif playlist_id == 'wishlist':
                    # Extract spotify_data again since it might be buried
                    spotify_data = track_info.get('spotify_data')
                    if isinstance(spotify_data, str):
                        try:
                            import json
                            spotify_data = json.loads(spotify_data)
                        except:
                            spotify_data = {}
                    
                    if not spotify_data:
                        spotify_data = {}

                    s_album = spotify_data.get('album') or {}
                    s_artists = spotify_data.get('artists', [])

                    # We need at least an album name and artist
                    if s_album and s_album.get('name'):
                        # Construct artist context for folder path.
                        # Priority: 1) watchlist artist name (the artist the user monitors)
                        #           2) album.artists[0]  3) track artists
                        # Watchlist artist is preferred because collab singles have combined
                        # album artists like "Y2K & Ro Ransom" — but the user is monitoring
                        # just "Y2K", so the folder should be "Y2K".
                        artist_ctx = {}
                        s_album_artists = s_album.get('artists', [])
                        source_info = track_info.get('source_info') or {}
                        if isinstance(source_info, str):
                            try:
                                import json
                                source_info = json.loads(source_info)
                            except:
                                source_info = {}

                        if source_info.get('watchlist_artist_name'):
                            # Best for watchlist tracks: the individual artist being monitored
                            artist_ctx = {'name': source_info['watchlist_artist_name'],
                                          'id': source_info.get('watchlist_artist_id', '')}
                        elif source_info.get('artist_name'):
                            # Other sources that set an explicit artist name
                            artist_ctx = {'name': source_info['artist_name']}
                        elif s_album_artists and len(s_album_artists) > 0:
                            # Album-level artists from Spotify (may be combined for collabs)
                            first_artist = s_album_artists[0]
                            if isinstance(first_artist, dict):
                                artist_ctx = first_artist
                            else:
                                artist_ctx = {'name': str(first_artist)}
                        elif s_artists and len(s_artists) > 0:
                            # Last resort: track-level artist (old wishlist entries without album artist data)
                            first_artist = s_artists[0]
                            if isinstance(first_artist, dict):
                                artist_ctx = first_artist
                            else:
                                artist_ctx = {'name': str(first_artist)}
                        else:
                            # Fallback if no artist at all
                            artist_ctx = {'name': track_info.get('artist', 'Unknown Artist')}

                        # Construct minimal album context
                        # Ensure images are preserved (important for artwork)
                        album_id = s_album.get('id', 'wishlist_album')
                        album_ctx = {
                            'id': album_id,
                            'name': s_album.get('name'),
                            'release_date': s_album.get('release_date', ''),
                            'total_tracks': s_album.get('total_tracks', 1),
                            'total_discs': wishlist_album_disc_counts.get(album_id, 1),
                            'album_type': s_album.get('album_type', 'album'),
                            'images': s_album.get('images', []) # Pass images array directly
                        }

                        track_info['_explicit_album_context'] = album_ctx
                        track_info['_explicit_artist_context'] = artist_ctx
                        track_info['_is_explicit_album_download'] = True
                        print(f"🎵 [Wishlist] Added album context for: '{track_info.get('name')}' -> '{album_ctx['name']}'")


                # Add playlist folder mode flag for sync page playlists
                if batch_playlist_folder_mode:
                    track_info['_playlist_folder_mode'] = True
                    track_info['_playlist_name'] = batch_playlist_name
                    print(f"📁 [Task Creation] Added playlist folder mode for: {track_info.get('name')} → {batch_playlist_name}")
                else:
                    print(f"🔍 [Debug] Task Creation - playlist folder mode NOT enabled for: {track_info.get('name')}")

                download_tasks[task_id] = {
                    'status': 'pending', 'track_info': track_info,
                    'playlist_id': playlist_id, 'batch_id': batch_id,
                    'track_index': res['track_index'], 'retry_count': 0,
                    'cached_candidates': [], 'used_sources': set(),
                    'status_change_time': time.time(),
                    'metadata_enhanced': False
                }
                download_batches[batch_id]['queue'].append(task_id)

        download_monitor.start_monitoring(batch_id)
        _start_next_batch_of_downloads(batch_id)

    except Exception as e:
        print(f"❌ Master worker for batch {batch_id} failed: {e}")
        import traceback
        traceback.print_exc()

        is_auto_batch = False
        with tasks_lock:
            if batch_id in download_batches:
                is_auto_batch = download_batches[batch_id].get('auto_initiated', False)
                download_batches[batch_id]['phase'] = 'error'
                download_batches[batch_id]['error'] = str(e)

                # Reset YouTube playlist phase to 'discovered' if this is a YouTube playlist on error
                if playlist_id.startswith('youtube_'):
                    url_hash = playlist_id.replace('youtube_', '')
                    if url_hash in youtube_playlist_states:
                        youtube_playlist_states[url_hash]['phase'] = 'discovered'
                        print(f"📋 Reset YouTube playlist {url_hash} to discovered phase (error)")

        # Handle auto-initiated wishlist errors - reset flag
        if is_auto_batch and playlist_id == 'wishlist':
            print("❌ [Auto-Wishlist] Master worker error - resetting auto-processing flag")
            global wishlist_auto_processing, wishlist_auto_processing_timestamp
            with wishlist_timer_lock:
                wishlist_auto_processing = False
                wishlist_auto_processing_timestamp = 0

def _run_post_processing_worker(task_id, batch_id):
    """
    NEW VERIFICATION WORKFLOW: Post-processing worker that only sets 'completed' status
    after successful file verification and processing. This matches sync.py's reliability.
    """
    try:
        print(f"🔧 [Post-Processing] Starting verification for task {task_id}")
        
        # Retrieve task details from global state
        with tasks_lock:
            if task_id not in download_tasks:
                print(f"❌ [Post-Processing] Task {task_id} not found in download_tasks")
                return
            task = download_tasks[task_id].copy()
            
        # Check if task was cancelled or already completed during post-processing
        if task['status'] == 'cancelled':
            print(f"❌ [Post-Processing] Task {task_id} was cancelled, skipping verification")
            return
        if task['status'] == 'completed' or task.get('stream_processed'):
            print(f"✅ [Post-Processing] Task {task_id} already completed by stream processor, skipping verification")
            return
            
        # Extract file information for verification
        track_info = task.get('track_info', {})
        task_filename = task.get('filename') or track_info.get('filename')
        task_username = task.get('username') or track_info.get('username')
        
        if not task_filename or not task_username:
            print(f"❌ [Post-Processing] Missing filename or username for task {task_id}")
            with tasks_lock:
                if task_id in download_tasks:
                    download_tasks[task_id]['status'] = 'failed'
                    download_tasks[task_id]['error_message'] = 'Post-processing failed: missing file or source information from Soulseek transfer'
            _on_download_completed(batch_id, task_id, success=False)
            return
            
        download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
        transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer'))
        
        # Try to get context for generating the correct final filename
        task_basename = extract_filename(task_filename)
        context_key = _make_context_key(task_username, task_filename)
        expected_final_filename = None
        
        print(f"🔍 [Post-Processing] Looking up context with key: {context_key}")
        
        with matched_context_lock:
            context = matched_downloads_context.get(context_key)
            # Debug: Show all available context keys
            available_keys = list(matched_downloads_context.keys())
            print(f"🔍 [Post-Processing] Available context keys: {available_keys[:10]}...")  # Show first 10 keys
            
        if context:
            print(f"✅ [Post-Processing] Found context for key: {context_key}")
            try:
                original_search = context.get("original_search_result", {})
                print(f"🔍 [Post-Processing] original_search keys: {list(original_search.keys())}")
                
                spotify_clean_title = original_search.get('spotify_clean_title')
                track_number = original_search.get('track_number')
                
                print(f"🔍 [Post-Processing] spotify_clean_title: '{spotify_clean_title}', track_number: {track_number}")
                
                if spotify_clean_title and track_number:
                    # Generate expected final filename that stream processor would create
                    # Pattern: f"{track_number:02d} - {clean_title}.flac"
                    sanitized_title = spotify_clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_')
                    expected_final_filename = f"{track_number:02d} - {sanitized_title}.flac"
                    print(f"🎯 [Post-Processing] Generated expected final filename: {expected_final_filename}")
                else:
                    print(f"❌ [Post-Processing] Missing required data - spotify_clean_title: {bool(spotify_clean_title)}, track_number: {bool(track_number)}")
            except Exception as e:
                print(f"⚠️ [Post-Processing] Error generating expected filename: {e}")
                import traceback
                traceback.print_exc()
        else:
            print(f"❌ [Post-Processing] No context found for key: {context_key}")
            # Try fuzzy matching with similar keys containing the filename
            # SAFETY: Constrain to same Soulseek username to prevent cross-album
            # metadata contamination during mass downloads (e.g., two albums both
            # having "01 - Intro.flac" would match the wrong context without this)
            with matched_context_lock:
                similar_keys = [k for k in matched_downloads_context.keys()
                                if k.startswith(f"{task_username}::") and task_basename in k]
            if similar_keys:
                # Use the first similar key found
                fuzzy_key = similar_keys[0]
                context = matched_downloads_context.get(fuzzy_key)
                print(f"✅ [Post-Processing] Found context using fuzzy key matching: {fuzzy_key}")
                
                # Generate expected final filename using the found context
                try:
                    original_search = context.get("original_search_result", {})
                    print(f"🔍 [Post-Processing] fuzzy context original_search keys: {list(original_search.keys())}")
                    
                    spotify_clean_title = original_search.get('spotify_clean_title')
                    track_number = original_search.get('track_number')
                    
                    print(f"🔍 [Post-Processing] fuzzy context spotify_clean_title: '{spotify_clean_title}', track_number: {track_number}")
                    
                    if spotify_clean_title and track_number:
                        # Generate expected final filename that stream processor would create
                        # Pattern: f"{track_number:02d} - {clean_title}.flac"
                        sanitized_title = spotify_clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_')
                        expected_final_filename = f"{track_number:02d} - {sanitized_title}.flac"
                        print(f"🎯 [Post-Processing] Generated expected final filename from fuzzy match: {expected_final_filename}")
                    else:
                        print(f"❌ [Post-Processing] Missing required data from fuzzy match - spotify_clean_title: {bool(spotify_clean_title)}, track_number: {bool(track_number)}")
                except Exception as e:
                    print(f"⚠️ [Post-Processing] Error generating expected filename from fuzzy match: {e}")
                    import traceback
                    traceback.print_exc()
            else:
                print(f"🔍 [Post-Processing] No similar keys found containing '{task_basename}'")
                # Show a sample of what keys actually exist for debugging
                sample_keys = list(matched_downloads_context.keys())[:5]
                print(f"🔍 [Post-Processing] Sample of existing keys: {sample_keys}")
        
        # RESILIENT FILE-FINDING LOOP: Try up to 3 times with delays
        found_file = None
        file_location = None

        # CRITICAL FIX: For YouTube downloads, the filename in task is 'id||title' (metadata),
        # but the actual file on disk is 'Title.mp3'. We must ask the client for the real path.
        if (task.get('username') == 'youtube' or '||' in str(task_filename)) and not found_file:
            logger.info(f"🔧 [Post-Processing] Detected YouTube download task: {task_id}")
            try:
                # Query the download orchestrator for the status which contains the real file path
                # CRITICAL FIX: Use the actual download_id designated by the client, not the internal task_id
                actual_download_id = task.get('download_id') or task_id
                status = run_async(soulseek_client.get_download_status(actual_download_id))
                if status and status.file_path:
                    real_path = status.file_path
                    if os.path.exists(real_path):
                        # Determine if it's in download or transfer directory
                        real_path_obj = Path(real_path)
                        download_dir_obj = Path(download_dir)
                        transfer_dir_obj = Path(transfer_dir)
                        
                        # Use absolute path comparison
                        try:
                            if download_dir_obj.resolve() in real_path_obj.resolve().parents:
                                file_location = 'download'
                            elif transfer_dir_obj.resolve() in real_path_obj.resolve().parents:
                                file_location = 'transfer'
                            else:
                                file_location = 'absolute'
                        except:
                            # Fallback if resolve fails (e.g. permission or path issues)
                            file_location = 'absolute'

                        if file_location:
                            # We found the file! Use the absolute path if it confuses the joining logic,
                            # but usually we want just the filename if location is 'download'/'transfer'
                            # CRITICAL FIX: Always use the absolute real_path.
                            # Stripping to basename causes FileNotFoundError because post-processing
                            # runs with CWD as project root, not download dir.
                            found_file = real_path

                            logger.info(f"✅ [Post-Processing] Resolved actual YouTube filename: {found_file} (Location: {file_location})")
                    else:
                        logger.warning(f"⚠️ [Post-Processing] YouTube status reported path but file missing: {real_path}")
                else:
                    logger.warning(f"⚠️ [Post-Processing] YouTube status returned no file_path for task {task_id}")
            except Exception as e:
                logger.error(f"⚠️ [Post-Processing] Failed to retrieve YouTube task status: {e}")

        _file_search_max_retries = 5
        for retry_count in range(_file_search_max_retries):
            # If we already resolved the file (e.g. via YouTube status), skip searching
            if found_file:
                print(f"🎯 [Post-Processing] Skipping search loop, file already resolved: {found_file}")
                break

            # Check if stream processor already completed this task while we were waiting
            with tasks_lock:
                if task_id in download_tasks:
                    if download_tasks[task_id].get('stream_processed') or download_tasks[task_id]['status'] == 'completed':
                        print(f"✅ [Post-Processing] Task {task_id} was completed by stream processor during file search - done")
                        return

            print(f"🔍 [Post-Processing] Attempt {retry_count + 1}/{_file_search_max_retries} to find file")
            print(f"🔍 [Post-Processing] Original filename: {task_basename}")
            if expected_final_filename:
                print(f"🔍 [Post-Processing] Expected final filename: {expected_final_filename}")
            else:
                print(f"⚠️ [Post-Processing] No expected final filename available")

            # Strategy 1: Try with original filename in both downloads and transfer
            print(f"🔍 [Post-Processing] Strategy 1: Searching with original filename...")
            found_file, file_location = _find_completed_file_robust(download_dir, task_filename, transfer_dir)

            if found_file:
                print(f"✅ [Post-Processing] Strategy 1 SUCCESS: Found file with original filename in {file_location}: {found_file}")
            else:
                print(f"❌ [Post-Processing] Strategy 1 FAILED: Original filename not found in either location")

            # Strategy 2: If not found and we have an expected final filename, try that in transfer folder
            if not found_file and expected_final_filename:
                print(f"🔍 [Post-Processing] Strategy 2: Searching transfer folder with expected final filename...")
                found_result = _find_completed_file_robust(transfer_dir, expected_final_filename)
                if found_result and found_result[0]:
                    found_file, file_location = found_result[0], 'transfer'
                    print(f"✅ [Post-Processing] Strategy 2 SUCCESS: Found file with expected final filename: {found_file}")
                else:
                    print(f"❌ [Post-Processing] Strategy 2 FAILED: Expected final filename not found in transfer folder")
            elif not expected_final_filename:
                print(f"⏭️ [Post-Processing] Strategy 2 SKIPPED: No expected final filename available")

            if found_file:
                print(f"🎯 [Post-Processing] FILE FOUND after {retry_count + 1} attempts in {file_location}: {found_file}")
                break
            else:
                print(f"❌ [Post-Processing] All search strategies failed on attempt {retry_count + 1}/{_file_search_max_retries}")
                if retry_count < _file_search_max_retries - 1:  # Don't sleep on final attempt
                    print(f"⏳ [Post-Processing] Waiting 5 seconds before next attempt...")
                    time.sleep(5)

        if not found_file:
            # CRITICAL: Before marking as failed, check if stream processor already handled this
            # The /api/downloads/status polling endpoint processes files independently and may have
            # already moved/renamed/tagged the file successfully while we were searching
            with tasks_lock:
                if task_id in download_tasks:
                    if download_tasks[task_id].get('stream_processed') or download_tasks[task_id]['status'] == 'completed':
                        print(f"✅ [Post-Processing] Task {task_id} was completed by stream processor - not marking as failed")
                        return
                    download_tasks[task_id]['status'] = 'failed'
                    download_tasks[task_id]['error_message'] = f'File not found on disk after {_file_search_max_retries} search attempts. Expected: {os.path.basename(task_filename)}'
            _on_download_completed(batch_id, task_id, success=False)
            return
            
        # Handle file found in transfer folder - already completed by stream processor
        if file_location == 'transfer':
            print(f"🎯 [Post-Processing] File found in transfer folder - already completed by stream processor: {found_file}")
            
            # Check if metadata enhancement was completed
            metadata_enhanced = False
            with tasks_lock:
                if task_id in download_tasks:
                    metadata_enhanced = download_tasks[task_id].get('metadata_enhanced', False)
            
            if not metadata_enhanced:
                print(f"⚠️ [Post-Processing] File in transfer folder missing metadata enhancement - completing now")
                # Attempt to complete metadata enhancement using context
                if context and expected_final_filename:
                    try:
                        # Extract required data from context
                        original_search = context.get("original_search_result", {})
                        spotify_artist = context.get("spotify_artist")
                        spotify_album = context.get("spotify_album")
                        
                        if spotify_artist and spotify_album:
                            # CRITICAL FIX: Create album_info dict with proper structure for metadata enhancement
                            # This must match the format used in main stream processor to ensure consistency
                            
                            # Extract track number from context (should be available from fuzzy match)
                            original_search = context.get("original_search_result", {})
                            track_number = original_search.get('track_number', 1)
                            
                            # If no track number in context, extract from filename 
                            if track_number == 1 and found_file:
                                print(f"⚠️ [Verification] No track_number in context, extracting from filename: {os.path.basename(found_file)}")
                                track_number = _extract_track_number_from_filename(found_file)
                                print(f"   -> Extracted track number: {track_number}")
                            
                            # Ensure track_number is valid
                            if not isinstance(track_number, int) or track_number < 1:
                                print(f"⚠️ [Verification] Invalid track number ({track_number}), defaulting to 1")
                                track_number = 1
                            
                            # Get clean track name 
                            clean_track_name = (original_search.get('spotify_clean_title') or 
                                              original_search.get('title', 'Unknown Track'))
                            
                            album_info = {
                                'is_album': True,  # CRITICAL: Mark as album track
                                'album_name': spotify_album.get('name', 'Unknown Album'),  # CORRECT KEY
                                'track_number': track_number,  # CORRECTED TRACK NUMBER
                                'disc_number': original_search.get('disc_number', 1),
                                'clean_track_name': clean_track_name,
                                'album_image_url': spotify_album.get('images', [{}])[0].get('url') if spotify_album.get('images') else None,
                                'confidence': 0.9,
                                'source': 'verification_worker_corrected'
                            }

                            # Apply album grouping for consistency with stream processor path.
                            # Only for singles/auto-detected — explicit album downloads already
                            # have the correct Spotify name and re-grouping would mangle it.
                            if not context.get("is_album_download", False):
                                try:
                                    raw_album_ctx = original_search.get('album')
                                    if isinstance(raw_album_ctx, str):
                                        original_album_ctx = raw_album_ctx
                                    elif isinstance(raw_album_ctx, dict) and 'name' in raw_album_ctx:
                                        original_album_ctx = raw_album_ctx['name']
                                    else:
                                        original_album_ctx = None
                                    consistent_album_name = _resolve_album_group(spotify_artist, album_info, original_album_ctx)
                                    album_info['album_name'] = consistent_album_name
                                except Exception as group_err:
                                    print(f"⚠️ [Verification] Album grouping failed, using raw name: {group_err}")
                            else:
                                print(f"🎯 [Verification] Explicit album download - preserving Spotify album name: '{album_info['album_name']}'")

                            print(f"🎯 [Verification] Created proper album_info - track_number: {track_number}, album: {album_info['album_name']}")

                            print(f"🎵 [Post-Processing] Attempting metadata enhancement for: {found_file}")
                            print(f"🔍 [Metadata Input] Verification worker - artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})")
                            print(f"🔍 [Metadata Input] Verification worker - album: '{album_info.get('album_name', 'MISSING')}', track#: {album_info.get('track_number', 'MISSING')}, source: {album_info.get('source', 'unknown')}")
                            enhancement_success = _enhance_file_metadata(found_file, context, spotify_artist, album_info)

                            if enhancement_success:
                                with tasks_lock:
                                    if task_id in download_tasks:
                                        download_tasks[task_id]['metadata_enhanced'] = True
                                print(f"✅ [Post-Processing] Successfully completed metadata enhancement for: {os.path.basename(found_file)}")
                            else:
                                print(f"⚠️ [Post-Processing] Metadata enhancement returned False for: {os.path.basename(found_file)}")
                        else:
                            print(f"⚠️ [Post-Processing] Missing spotify_artist or spotify_album in context")
                            print(f"⚠️ [Post-Processing] spotify_artist: {spotify_artist is not None}, spotify_album: {spotify_album is not None}")
                    except Exception as enhancement_error:
                        import traceback
                        print(f"❌ [Post-Processing] Error during metadata enhancement: {enhancement_error}\n{traceback.format_exc()}")
                else:
                    print(f"⚠️ [Post-Processing] Cannot complete metadata enhancement - missing context or expected filename")
            else:
                print(f"✅ [Post-Processing] File already has metadata enhancement completed")
            
            with tasks_lock:
                if task_id in download_tasks:
                    track_info = download_tasks[task_id].get('track_info')
                    _mark_task_completed(task_id, track_info)
            
            # Clean up context now that both stream processor and verification worker are done
            with matched_context_lock:
                if context_key in matched_downloads_context:
                    del matched_downloads_context[context_key]
                    print(f"🗑️ [Verification] Cleaned up context after successful verification: {context_key}")
            
            _on_download_completed(batch_id, task_id, success=True)
            return
            
        # File found in downloads folder - attempt post-processing
        try:
            # Rebuild the context key using the same function that stored it
            context_key = _make_context_key(task_username, task_filename)

            # Check if this download has matched context for post-processing
            with matched_context_lock:
                context = matched_downloads_context.get(context_key)
                
            if context:
                print(f"🎯 [Post-Processing] Found matched context, running full post-processing for: {context_key}")
                # Run the existing post-processing logic with verification
                _post_process_matched_download_with_verification(context_key, context, found_file, task_id, batch_id)
            else:
                # No matched context - just mark as completed since file exists
                print(f"📁 [Post-Processing] No matched context, marking as completed: {os.path.basename(found_file)}")
                with tasks_lock:
                    if task_id in download_tasks:
                        track_info = download_tasks[task_id].get('track_info')
                        _mark_task_completed(task_id, track_info)
                
                # Clean up context if it exists (might be leftover from stream processor)
                with matched_context_lock:
                    if context_key in matched_downloads_context:
                        del matched_downloads_context[context_key]
                        print(f"🗑️ [Verification] Cleaned up leftover context: {context_key}")
                
                # Call completion callback since there's no other post-processing to handle it
                _on_download_completed(batch_id, task_id, success=True)
                
        except Exception as processing_error:
            print(f"❌ [Post-Processing] Processing failed for task {task_id}: {processing_error}")
            with tasks_lock:
                if task_id in download_tasks:
                    download_tasks[task_id]['status'] = 'failed'
                    download_tasks[task_id]['error_message'] = f"Post-processing failed: {str(processing_error)}"
            _on_download_completed(batch_id, task_id, success=False)
            
    except Exception as e:
        print(f"❌ [Post-Processing] Critical error in post-processing worker for task {task_id}: {e}")
        with tasks_lock:
            if task_id in download_tasks:
                download_tasks[task_id]['status'] = 'failed'
                download_tasks[task_id]['error_message'] = f"Critical post-processing error: {str(e)}"
        _on_download_completed(batch_id, task_id, success=False)


def _download_track_worker(task_id, batch_id=None):
    """
    Enhanced download worker that matches the GUI's exact retry logic.
    Implements sequential query retry, fallback candidates, and download failure retry.
    """
    try:
        # Retrieve task details from global state
        with tasks_lock:
            if task_id not in download_tasks:
                print(f"❌ [Modal Worker] Task {task_id} not found in download_tasks")
                return
            task = download_tasks[task_id].copy()
            
        # Cancellation Checkpoint 1: Before doing anything
        with tasks_lock:
            if task_id not in download_tasks:
                print(f"❌ [Modal Worker] Task {task_id} was deleted before starting")
                return
            if download_tasks[task_id]['status'] == 'cancelled':
                print(f"❌ [Modal Worker] Task {task_id} cancelled before starting")
                # V2 FIX: Don't call _on_download_completed for cancelled V2 tasks
                # V2 system handles worker slot freeing in atomic cancel function
                task_playlist_id = download_tasks[task_id].get('playlist_id')
                if task_playlist_id:
                    print(f"⏭️ [Modal Worker] V2 task {task_id} cancelled - worker slot already freed by V2 system")
                    return  # V2 system already handled worker slot management
                elif batch_id:
                    # Legacy system - use old completion callback
                    print(f"⏭️ [Modal Worker] Legacy task {task_id} cancelled - using legacy completion callback")
                    _on_download_completed(batch_id, task_id, success=False)
                return

        track_data = task['track_info']
        track_name = track_data.get('name', 'Unknown Track')
        
        print(f"🎯 [Modal Worker] Task {task_id} starting search for track: '{track_name}'")
        
        # Recreate a SpotifyTrack object for the matching engine
        # Handle both string format and Spotify API format for artists
        raw_artists = track_data.get('artists', [])
        processed_artists = []
        for artist in raw_artists:
            if isinstance(artist, str):
                processed_artists.append(artist)
            elif isinstance(artist, dict) and 'name' in artist:
                processed_artists.append(artist['name'])
            else:
                processed_artists.append(str(artist))
        
        # Handle album field - extract name if it's a dictionary
        raw_album = track_data.get('album', '')
        if isinstance(raw_album, dict) and 'name' in raw_album:
            album_name = raw_album['name']
        elif isinstance(raw_album, str):
            album_name = raw_album
        else:
            album_name = str(raw_album)
        
        track = SpotifyTrack(
            id=track_data.get('id', ''),
            name=track_data.get('name', ''),
            artists=processed_artists,
            album=album_name,
            duration_ms=track_data.get('duration_ms', 0),
            popularity=track_data.get('popularity', 0)
        )
        print(f"📥 [Modal Worker] Starting download task for: {track.name} by {track.artists[0] if track.artists else 'Unknown'}")

        # === SOURCE REUSE: Check batch's last good source before searching ===
        if _try_source_reuse(task_id, batch_id, track):
            # Store source for next worker (cascading reuse)
            with tasks_lock:
                used_filename = download_tasks.get(task_id, {}).get('filename')
                used_username = download_tasks.get(task_id, {}).get('username')
            if used_filename and used_username:
                _store_batch_source(batch_id, used_username, used_filename)
            return

        # Initialize task state tracking (like GUI's parallel_search_tracking)
        with tasks_lock:
            if task_id in download_tasks:
                download_tasks[task_id]['status'] = 'searching'  # Now actively being processed
                download_tasks[task_id]['current_query_index'] = 0
                download_tasks[task_id]['current_candidate_index'] = 0
                download_tasks[task_id]['retry_count'] = 0
                download_tasks[task_id]['candidates'] = []
                # CRITICAL: Preserve used_sources from previous retry attempts (don't reset to empty set)
                # If this is a retry, the monitor will have already marked failed sources
                if 'used_sources' not in download_tasks[task_id]:
                    download_tasks[task_id]['used_sources'] = set()
                # Else: keep existing used_sources to avoid retrying same failed hosts

        # 1. Generate multiple search queries (like GUI's generate_smart_search_queries)
        artist_name = track.artists[0] if track.artists else None
        track_name = track.name
        
        # Start with matching engine queries
        search_queries = matching_engine.generate_download_queries(track)
        
        # Add legacy fallback queries (like GUI does)
        legacy_queries = []
        
        if artist_name:
            # Add first word of artist approach (legacy compatibility)
            artist_words = artist_name.split()
            if artist_words:
                first_word = artist_words[0]
                if first_word.lower() == 'the' and len(artist_words) > 1:
                    first_word = artist_words[1]
                
                if len(first_word) > 1:
                    legacy_queries.append(f"{track_name} {first_word}".strip())
        
        # Add track-only query
        if track_name.strip():
            legacy_queries.append(track_name.strip())
        
        # Add traditional cleaned queries
        cleaned_name = re.sub(r'\s*\([^)]*\)', '', track_name).strip()
        cleaned_name = re.sub(r'\s*\[[^\]]*\]', '', cleaned_name).strip()
        
        if cleaned_name and cleaned_name.lower() != track_name.lower():
            legacy_queries.append(cleaned_name.strip())
        
        # Combine enhanced queries with legacy fallbacks
        all_queries = search_queries + legacy_queries
        
        # Remove duplicates while preserving order
        unique_queries = []
        seen = set()
        for query in all_queries:
            if query and query.lower() not in seen:
                unique_queries.append(query)
                seen.add(query.lower())
        
        search_queries = unique_queries
        print(f"🔍 [Modal Worker] Generated {len(search_queries)} smart search queries for '{track.name}': {search_queries}")
        print(f"🔍 [Modal Worker] About to start search loop for task {task_id} (track: '{track.name}')")

        # 2. Sequential Query Search (matches GUI's start_search_worker_parallel logic)
        search_diagnostics = []  # Track what happened per query for detailed error messages
        all_raw_results = []  # Collect raw results across queries for candidate review modal
        for query_index, query in enumerate(search_queries):
            # Cancellation check before each query
            with tasks_lock:
                if task_id not in download_tasks:
                    print(f"❌ [Modal Worker] Task {task_id} was deleted during query {query_index + 1}")
                    return
                if download_tasks[task_id]['status'] == 'cancelled':
                    print(f"❌ [Modal Worker] Task {task_id} cancelled during query {query_index + 1}")
                    # Don't call _on_download_completed for cancelled tasks as it can stop monitoring
                    return
                download_tasks[task_id]['current_query_index'] = query_index
                    
            print(f"🔍 [Modal Worker] Query {query_index + 1}/{len(search_queries)}: '{query}'")
            print(f"🔍 [DEBUG] About to call soulseek search for task {task_id}")
            
            try:
                # Perform search with timeout
                tracks_result, _ = run_async(soulseek_client.search(query, timeout=30))
                print(f"🔍 [DEBUG] Search completed for task {task_id}, got {len(tracks_result) if tracks_result else 0} results")
                
                # CRITICAL: Check cancellation immediately after search returns
                with tasks_lock:
                    if task_id not in download_tasks:
                        print(f"❌ [Modal Worker] Task {task_id} was deleted after search returned")
                        return
                    if download_tasks[task_id]['status'] == 'cancelled':
                        print(f"❌ [Modal Worker] Task {task_id} cancelled after search returned - ignoring results")
                        # Don't call _on_download_completed for cancelled tasks as it can stop monitoring
                        # The cancellation endpoint already handles batch management properly
                        return
                
                if tracks_result:
                    result_count = len(tracks_result)
                    # Validate candidates using GUI's get_valid_candidates logic
                    candidates = get_valid_candidates(tracks_result, track, query)
                    if candidates:
                        print(f"✅ [Modal Worker] Found {len(candidates)} valid candidates for query '{query}'")

                        # CRITICAL: Check cancellation before processing candidates
                        with tasks_lock:
                            if task_id not in download_tasks:
                                print(f"❌ [Modal Worker] Task {task_id} was deleted before processing candidates")
                                return
                            if download_tasks[task_id]['status'] == 'cancelled':
                                print(f"❌ [Modal Worker] Task {task_id} cancelled before processing candidates")
                                # Don't call _on_download_completed for cancelled tasks as it can stop monitoring
                                return
                            # Store candidates for retry fallback (like GUI)
                            download_tasks[task_id]['cached_candidates'] = candidates

                        # Try to download with these candidates
                        success = _attempt_download_with_candidates(task_id, candidates, track, batch_id)
                        if success:
                            # Download initiated successfully - let the download monitoring system handle completion
                            if batch_id:
                                print(f"✅ [Modal Worker] Download initiated successfully for task {task_id} - monitoring will handle completion")
                            # Store this source for batch reuse
                            with tasks_lock:
                                used_filename = download_tasks.get(task_id, {}).get('filename')
                                used_username = download_tasks.get(task_id, {}).get('username')
                            if used_filename and used_username:
                                _store_batch_source(batch_id, used_username, used_filename)
                            return  # Success, exit the worker
                        else:
                            search_diagnostics.append(f'"{query}": {result_count} results, {len(candidates)} passed filters but download failed to start')
                    else:
                        search_diagnostics.append(f'"{query}": {result_count} results but none passed quality/artist filters')
                        all_raw_results.extend(tracks_result[:20])  # Keep top results for review
                else:
                    search_diagnostics.append(f'"{query}": no results on Soulseek')

            except Exception as e:
                print(f"⚠️ [Modal Worker] Search failed for query '{query}': {e}")
                search_diagnostics.append(f'"{query}": search error — {e}')
                continue

        # If we get here, all search queries failed
        print(f"❌ [Modal Worker] No valid candidates found for '{track.name}' after trying all {len(search_queries)} queries.")
        with tasks_lock:
            if task_id in download_tasks:
                download_tasks[task_id]['status'] = 'not_found'
                _diag_summary = ' | '.join(search_diagnostics) if search_diagnostics else 'no queries attempted'
                download_tasks[task_id]['error_message'] = f'No match found for "{track_name}" by {artist_name or "Unknown"} after {len(search_queries)} queries. Breakdown: {_diag_summary}'
                # Store raw results so the user can review what Soulseek returned
                if all_raw_results and not download_tasks[task_id].get('cached_candidates'):
                    download_tasks[task_id]['cached_candidates'] = all_raw_results
        
        # Notify batch manager that this task completed (failed) - THREAD SAFE
        if batch_id:
            try:
                _on_download_completed(batch_id, task_id, success=False)
            except Exception as completion_error:
                print(f"❌ Error in batch completion callback for {task_id}: {completion_error}")

    except Exception as e:
        import traceback
        track_name_safe = locals().get('track_name', 'unknown')  # Safe fallback for track_name
        print(f"❌ CRITICAL ERROR in download task for '{track_name_safe}' (task_id: {task_id}): {e}")
        traceback.print_exc()
        
        # Update task status safely with timeout
        try:
            lock_acquired = tasks_lock.acquire(timeout=2.0)
            if lock_acquired:
                try:
                    if task_id in download_tasks:
                        download_tasks[task_id]['status'] = 'failed'
                        download_tasks[task_id]['error_message'] = f'Unexpected error during download: {type(e).__name__}: {e}'
                        print(f"🔧 [Exception Recovery] Set task {task_id} status to 'failed'")
                finally:
                    tasks_lock.release()
            else:
                print(f"⚠️ [Exception Recovery] Could not acquire lock to update task {task_id} status")
        except Exception as status_error:
            print(f"❌ Error updating task status in exception handler: {status_error}")
        
        # Notify batch manager that this task completed (failed) - THREAD SAFE with RECOVERY
        if batch_id:
            try:
                _on_download_completed(batch_id, task_id, success=False)
                print(f"✅ [Exception Recovery] Successfully freed worker slot for task {task_id}")
            except Exception as completion_error:
                print(f"❌ [Exception Recovery] Error in batch completion callback for {task_id}: {completion_error}")
                # CRITICAL: If batch completion fails, we need to manually recover the worker slot
                try:
                    print(f"🚨 [Exception Recovery] Attempting manual worker slot recovery for batch {batch_id}")
                    _recover_worker_slot(batch_id, task_id)
                except Exception as recovery_error:
                    print(f"💀 [Exception Recovery] FATAL: Could not recover worker slot: {recovery_error}")

def _attempt_download_with_candidates(task_id, candidates, track, batch_id=None):
    """
    Attempts to download with fallback candidate logic (matches GUI's retry_parallel_download_with_fallback).
    Returns True if successful, False if all candidates fail.
    """
    # Sort candidates by confidence (best first)
    candidates.sort(key=lambda r: r.confidence, reverse=True)
    
    with tasks_lock:
        task = download_tasks.get(task_id)
        if not task:
            return False
        used_sources = task.get('used_sources', set())
    
    # Try each candidate until one succeeds (like GUI's fallback logic)
    for candidate_index, candidate in enumerate(candidates):
        # Check cancellation before each attempt
        with tasks_lock:
            if task_id not in download_tasks:
                print(f"❌ [Modal Worker] Task {task_id} was deleted during candidate {candidate_index + 1}")
                return False
            if download_tasks[task_id]['status'] == 'cancelled':
                print(f"❌ [Modal Worker] Task {task_id} cancelled during candidate {candidate_index + 1}")
                # Don't call _on_download_completed for cancelled tasks as it can stop monitoring
                return False
            download_tasks[task_id]['current_candidate_index'] = candidate_index
            
        # Create source key to avoid duplicate attempts (like GUI)
        source_key = f"{candidate.username}_{candidate.filename}"
        if source_key in used_sources:
            print(f"⏭️ [Modal Worker] Skipping already tried source: {source_key}")
            continue
        
        # CRITICAL: Add source to used_sources IMMEDIATELY to prevent race conditions
        # This must happen BEFORE starting download to prevent multiple retries from picking same source
        with tasks_lock:
            if task_id in download_tasks:
                download_tasks[task_id]['used_sources'].add(source_key)
                print(f"🚫 [Modal Worker] Marked source as used before download attempt: {source_key}")
            
        print(f"🎯 [Modal Worker] Trying candidate {candidate_index + 1}/{len(candidates)}: {candidate.filename} (Confidence: {candidate.confidence:.2f})")
        
        try:
            # Update task status to downloading
            _update_task_status(task_id, 'downloading')

            # Prepare download - check if we have explicit album context from artist page
            track_info = {}
            with tasks_lock:
                if task_id in download_tasks:
                    raw_track_info = download_tasks[task_id].get('track_info')
                    track_info = raw_track_info if isinstance(raw_track_info, dict) else {}

            # Use explicit album/artist context if available (from artist album downloads)
            has_explicit_context = track_info and track_info.get('_is_explicit_album_download', False)

            if has_explicit_context:
                # Use the real Spotify album/artist data from the UI
                explicit_album = track_info.get('_explicit_album_context', {})
                explicit_artist = track_info.get('_explicit_artist_context', {})
                # Normalize artist context if it's a plain string (e.g. from wishlist spotify_data)
                if isinstance(explicit_artist, str):
                    explicit_artist = {'name': explicit_artist}

                spotify_artist_context = {
                    'id': explicit_artist.get('id', 'explicit_artist'),
                    'name': explicit_artist.get('name', track.artists[0] if track.artists else 'Unknown'),
                    'genres': explicit_artist.get('genres', [])
                }
                # Handle both image_url formats (direct string or images array)
                album_image_url = None
                if explicit_album.get('image_url'):
                    # Backend API returns image_url as direct string
                    album_image_url = explicit_album.get('image_url')
                elif explicit_album.get('images'):
                    # Fallback: images array format from Spotify API
                    album_image_url = explicit_album.get('images', [{}])[0].get('url')

                spotify_album_context = {
                    'id': explicit_album.get('id', 'explicit_album'),
                    'name': explicit_album.get('name', track.album),
                    'release_date': explicit_album.get('release_date', ''),
                    'image_url': album_image_url,
                    'total_tracks': explicit_album.get('total_tracks', 0),
                    'total_discs': explicit_album.get('total_discs', 1),
                    'album_type': explicit_album.get('album_type', 'album')
                }
                print(f"🎵 [Explicit Context] Using real album data: '{spotify_album_context['name']}' ({spotify_album_context['album_type']}, {spotify_album_context['total_discs']} disc(s))")
            else:
                # Fallback to generic context for playlists/wishlists
                spotify_artist_context = {'id': 'from_sync_modal', 'name': track.artists[0] if track.artists else 'Unknown', 'genres': []}
                spotify_album_context = {'id': 'from_sync_modal', 'name': track.album, 'release_date': '', 'image_url': None}

            download_payload = candidate.__dict__

            username = download_payload.get('username')
            filename = download_payload.get('filename')
            size = download_payload.get('size', 0)

            if not username or not filename:
                print(f"❌ [Modal Worker] Invalid candidate data: missing username or filename")
                continue

            # PROTECTION: Check if there's already an active download for this task
            current_download_id = None
            with tasks_lock:
                if task_id in download_tasks:
                    current_download_id = download_tasks[task_id].get('download_id')
            
            if current_download_id:
                print(f"⚠️ [Modal Worker] Task {task_id} already has active download {current_download_id} - skipping new download attempt")
                print(f"🔄 [Modal Worker] This prevents race condition where multiple retries start overlapping downloads")
                continue

            # Initiate download
            print(f"🚀 [Modal Worker] Starting download: {username} / {os.path.basename(filename)}")
            download_id = run_async(soulseek_client.download(username, filename, size))

            if download_id:
                # Store context for post-processing with complete Spotify metadata (GUI PARITY)
                context_key = _make_context_key(username, filename)
                with matched_context_lock:
                    # Create WebUI equivalent of GUI's SpotifyBasedSearchResult data structure
                    enhanced_payload = download_payload.copy()
                    
                    # Extract clean Spotify metadata from track object (same as GUI)
                    has_clean_spotify_data = track and hasattr(track, 'name') and hasattr(track, 'album')
                    if has_clean_spotify_data:
                        # Use clean Spotify metadata (matches GUI's SpotifyBasedSearchResult)
                        enhanced_payload['spotify_clean_title'] = track.name
                        enhanced_payload['spotify_clean_album'] = track.album
                        enhanced_payload['spotify_clean_artist'] = track.artists[0] if track.artists else enhanced_payload.get('artist', '')
                        # Preserve all artists for metadata tagging
                        enhanced_payload['artists'] = [{'name': artist} for artist in track.artists] if track.artists else []
                        print(f"✨ [Context] Using clean Spotify metadata - Album: '{track.album}', Title: '{track.name}'")
                        
                        # CRITICAL FIX: Get track_number and disc_number from Spotify API like GUI does
                        if hasattr(track, 'id') and track.id:
                            try:
                                detailed_track = spotify_client.get_track_details(track.id)
                                if detailed_track and 'track_number' in detailed_track:
                                    enhanced_payload['track_number'] = detailed_track['track_number']
                                    enhanced_payload['disc_number'] = detailed_track.get('disc_number', 1)
                                    print(f"🔢 [Context] Added Spotify track_number: {detailed_track['track_number']}, disc_number: {enhanced_payload['disc_number']}")
                                else:
                                    enhanced_payload['track_number'] = track_info.get('track_number', 1)
                                    enhanced_payload['disc_number'] = track_info.get('disc_number', 1)
                                    print(f"⚠️ [Context] No track_number in detailed_track, using track_info fallback: {enhanced_payload['track_number']}")
                            except Exception as e:
                                enhanced_payload['track_number'] = track_info.get('track_number', 1)
                                enhanced_payload['disc_number'] = track_info.get('disc_number', 1)
                                print(f"❌ [Context] Error getting track_number, using track_info fallback: {enhanced_payload['track_number']} ({e})")
                        else:
                            enhanced_payload['track_number'] = track_info.get('track_number', 1)
                            enhanced_payload['disc_number'] = track_info.get('disc_number', 1)
                            print(f"⚠️ [Context] No track.id available, using track_info fallback track_number: {enhanced_payload['track_number']}")
                        
                        # Determine if this should be treated as album download
                        # First check if we have explicit album context from artist page
                        if has_explicit_context:
                            is_album_context = True
                            print(f"✅ [Context] Using explicit album context flag from artist page")
                        else:
                            # Fall back to guessing based on clean data
                            is_album_context = (
                                track.album and
                                track.album.strip() and
                                track.album != "Unknown Album" and
                                track.album.lower() != track.name.lower()  # Album different from track
                            )
                    else:
                        # Fallback to original data
                        enhanced_payload['spotify_clean_title'] = enhanced_payload.get('title', '')
                        enhanced_payload['spotify_clean_album'] = enhanced_payload.get('album', '')
                        enhanced_payload['spotify_clean_artist'] = enhanced_payload.get('artist', '')
                        # Preserve existing artists array if available, otherwise create from single artist
                        if 'artists' not in enhanced_payload and enhanced_payload.get('artist'):
                            enhanced_payload['artists'] = [{'name': enhanced_payload['artist']}]
                        enhanced_payload['track_number'] = track_info.get('track_number', 1)  # Fallback when no clean Spotify data
                        is_album_context = False
                        print(f"⚠️ [Context] Using fallback data - no clean Spotify metadata available, track_number={enhanced_payload['track_number']}")
                    
                    matched_downloads_context[context_key] = {
                        "spotify_artist": spotify_artist_context,
                        "spotify_album": spotify_album_context,
                        "original_search_result": enhanced_payload,
                        "is_album_download": is_album_context,  # Critical fix: Use actual album context
                        "has_clean_spotify_data": has_clean_spotify_data,  # Flag for post-processing
                        "task_id": task_id,  # Add task_id for completion callbacks
                        "batch_id": batch_id,  # Add batch_id for completion callbacks
                        "track_info": track_info  # Add track_info for playlist folder mode
                    }

                    print(f"🎯 [Context] Set is_album_download: {is_album_context} (has clean data: {has_clean_spotify_data})")
                    print(f"🔍 [Debug] Context creation - track_info: {track_info is not None}, playlist_folder_mode: {track_info.get('_playlist_folder_mode', False) if track_info else False}")
                
                # Update task with successful download info
                with tasks_lock:
                    if task_id in download_tasks:
                        # PHASE 3: Final cancellation check after download started (GUI PARITY)
                        if download_tasks[task_id]['status'] == 'cancelled':
                            print(f"🚫 [Modal Worker] Task {task_id} cancelled after download {download_id} started - attempting to cancel download")
                            # Try to cancel the download immediately
                            try:
                                run_async(soulseek_client.cancel_download(download_id, username, remove=True))
                                print(f"✅ Successfully cancelled active download {download_id}")
                            except Exception as cancel_error:
                                print(f"⚠️ Warning: Failed to cancel active download {download_id}: {cancel_error}")
                            
                            # Free worker slot
                            if batch_id:
                                _on_download_completed(batch_id, task_id, success=False)
                            return False
                        
                        # Store download information - use real download ID from soulseek_client
                        # CRITICAL FIX: Trust the download ID returned by soulseek_client.download()
                        download_tasks[task_id]['download_id'] = download_id
                        
                        download_tasks[task_id]['username'] = username
                        download_tasks[task_id]['filename'] = filename
                        
                print(f"✅ [Modal Worker] Download started successfully for '{filename}'. Download ID: {download_id}")
                return True  # Success!
            else:
                print(f"❌ [Modal Worker] Failed to start download for '{filename}'")
                # Reset status back to searching for next attempt
                with tasks_lock:
                    if task_id in download_tasks:
                        download_tasks[task_id]['status'] = 'searching'
                continue
                
        except Exception as e:
            import traceback
            print(f"❌ [Modal Worker] Error attempting download for '{candidate.filename}': {e}")
            traceback.print_exc()
            # Reset status back to searching for next attempt
            with tasks_lock:
                if task_id in download_tasks:
                    download_tasks[task_id]['status'] = 'searching'
            continue

    # All candidates failed
    print(f"❌ [Modal Worker] All {len(candidates)} candidates failed for '{track.name}'")
    return False

def _try_source_reuse(task_id, batch_id, track):
    """
    Check batch's last_good_source for the current track before searching.
    Returns True if source reuse succeeded, False to fall through to normal search.
    """
    _sr = source_reuse_logger
    _sr.info(f"_try_source_reuse called: task={task_id}, batch={batch_id}, track={track.name}")
    if not batch_id:
        _sr.info(f"Skipped — no batch_id")
        return False

    with tasks_lock:
        batch = download_batches.get(batch_id)
        if not batch:
            _sr.info(f"Skipped — batch {batch_id} not found")
            return False
        # Gate: album/EP downloads only
        is_album = batch.get('is_album_download', False)
        is_wishlist = batch.get('playlist_id', '') == 'wishlist'
        if not is_album and not is_wishlist:
            _sr.info(f"Skipped — not album ({is_album}) and not wishlist ({is_wishlist})")
            return False
        source_tracks = batch.get('source_folder_tracks')
        last_source = batch.get('last_good_source')
        _sr.info(f"Batch state: last_good_source={last_source}, source_folder_tracks={'None' if source_tracks is None else f'{len(source_tracks)} tracks'}")

    if not source_tracks or not last_source:
        _sr.info(f"Skipped — no source_tracks or no last_source")
        return False
    if last_source.get('username') == 'youtube':
        _sr.info(f"Skipped — youtube source")
        return False

    source_username = last_source.get('username')
    source_folder = last_source.get('folder_path', '')
    source_key = f"{source_username}:{source_folder}"

    # Check if this source+folder has already failed for this batch
    with tasks_lock:
        batch = download_batches.get(batch_id, {})
        failed_sources = batch.get('failed_sources', set())
    if source_key in failed_sources:
        _sr.info(f"Source {source_key} already in failed_sources — skipping")
        with tasks_lock:
            if batch_id in download_batches:
                download_batches[batch_id]['last_good_source'] = None
                download_batches[batch_id]['source_folder_tracks'] = None
        return False

    # Detect retry: if this task already tried the stored source and failed (monitor resubmitted),
    # the task will still have the previous username/download_id from the failed attempt
    with tasks_lock:
        task = download_tasks.get(task_id, {})
        prev_username = task.get('username')
        prev_download_id = task.get('download_id')

    if prev_username and prev_download_id and prev_username == source_username:
        _sr.info(f"Task {task_id} already failed from source {source_key} — marking as failed")
        with tasks_lock:
            if batch_id in download_batches:
                if 'failed_sources' not in download_batches[batch_id]:
                    download_batches[batch_id]['failed_sources'] = set()
                download_batches[batch_id]['failed_sources'].add(source_key)
                download_batches[batch_id]['last_good_source'] = None
                download_batches[batch_id]['source_folder_tracks'] = None
        return False

    _sr.info(f"Checking reused source for task {task_id}: {source_key}")

    # Score each folder track against current track
    candidates = []
    for folder_track in source_tracks:
        confidence = matching_engine.calculate_slskd_match_confidence(track, folder_track)
        _sr.info(f"  Match '{track.name}' vs '{folder_track.filename}' → confidence={confidence:.3f}")
        if confidence >= 0.70:
            folder_track.confidence = confidence
            candidates.append(folder_track)

    if not candidates:
        _sr.info(f"No folder tracks matched above 0.70 for task {task_id}")
        return False

    # Sort by confidence, filter by quality preference
    candidates.sort(key=lambda c: c.confidence, reverse=True)
    _sr.info(f"Found {len(candidates)} candidates above 0.70, best={candidates[0].confidence:.3f} ({candidates[0].filename})")
    slsk = soulseek_client.soulseek if hasattr(soulseek_client, 'soulseek') else soulseek_client
    filtered = slsk.filter_results_by_quality_preference(candidates)
    if not filtered:
        _sr.info(f"Quality filter rejected all candidates for task {task_id}")
        return False
    _sr.info(f"After quality filter: {len(filtered)} candidates remain")

    # Artist verification
    artist_name = track.artists[0].lower() if track.artists else ''
    verified = [c for c in filtered if artist_name and artist_name in c.filename.lower().replace('\\', '/')]
    final_candidates = verified if verified else filtered[:1]
    _sr.info(f"Artist verification: artist='{artist_name}', verified={len(verified)}, using={len(final_candidates)} candidates")

    # Initialize task state for download attempt
    # IMPORTANT: Preserve used_sources from previous attempts (e.g. monitor error retries)
    # so that _attempt_download_with_candidates skips sources that already failed.
    with tasks_lock:
        if task_id in download_tasks:
            download_tasks[task_id]['status'] = 'searching'
            download_tasks[task_id]['current_query_index'] = 0
            download_tasks[task_id]['current_candidate_index'] = 0
            download_tasks[task_id]['retry_count'] = 0
            download_tasks[task_id]['candidates'] = []
            # Don't reset used_sources — the download monitor marks failed sources here

    # Attempt download
    success = _attempt_download_with_candidates(task_id, final_candidates, track, batch_id)
    if success:
        _sr.info(f"SUCCESS — Downloaded from reused source for task {task_id}")
        return True

    # Source failed — mark as failed so it's never tried again for this batch
    with tasks_lock:
        if batch_id in download_batches:
            if 'failed_sources' not in download_batches[batch_id]:
                download_batches[batch_id]['failed_sources'] = set()
            download_batches[batch_id]['failed_sources'].add(source_key)
            download_batches[batch_id]['last_good_source'] = None
            download_batches[batch_id]['source_folder_tracks'] = None
    _sr.info(f"FAILED — Source {source_key} failed for task {task_id}, added to failed_sources")
    return False

def _store_batch_source(batch_id, username, filename):
    """Browse the successful download's folder and store results on the batch for reuse."""
    _sr = source_reuse_logger
    _sr.info(f"_store_batch_source called: batch={batch_id}, user={username}, file={filename}")
    if not batch_id or username in ('youtube', 'tidal'):
        _sr.info(f"Skipped — no batch_id or streaming source ({username})")
        return

    with tasks_lock:
        batch = download_batches.get(batch_id)
        if not batch:
            _sr.info(f"Skipped — batch not found")
            return
        is_album = batch.get('is_album_download', False)
        is_wishlist = batch.get('playlist_id', '') == 'wishlist'
        if not is_album and not is_wishlist:
            _sr.info(f"Skipped — not album ({is_album}) and not wishlist ({is_wishlist})")
            return
        # Don't store a source+folder that already failed for this batch
        failed_sources = batch.get('failed_sources', set())

    # Extract folder path from filename — preserve original separators for slskd API
    if '\\' in filename:
        folder_path = filename.rsplit('\\', 1)[0]
    elif '/' in filename:
        folder_path = filename.rsplit('/', 1)[0]
    else:
        _sr.info(f"Skipped — no folder separator in filename: {filename}")
        return

    # Check failed_sources with username:folder key
    source_key = f"{username}:{folder_path}"
    if source_key in failed_sources:
        _sr.info(f"Not storing source {source_key} — already in failed_sources")
        return

    try:
        # Access SoulseekClient directly (soulseek_client is DownloadOrchestrator)
        slsk = soulseek_client.soulseek if hasattr(soulseek_client, 'soulseek') else soulseek_client
        _sr.info(f"Browsing {username}:{folder_path}...")
        files = run_async(slsk.browse_user_directory(username, folder_path))
        if not files:
            _sr.info(f"Browse returned no files for {username}:{folder_path}")
            return
        _sr.info(f"Browse returned {len(files)} raw files")

        tracks = slsk.parse_browse_results_to_tracks(username, files, directory=folder_path)
        if not tracks:
            _sr.info(f"No audio tracks after parsing for {username}:{folder_path}")
            return
        _sr.info(f"Parsed {len(tracks)} audio tracks from {username}:{folder_path}")

        with tasks_lock:
            if batch_id in download_batches:
                download_batches[batch_id]['last_good_source'] = {
                    'username': username,
                    'folder_path': folder_path
                }
                download_batches[batch_id]['source_folder_tracks'] = tracks
                _sr.info(f"STORED {len(tracks)} tracks from {username}:{folder_path} as last_good_source")
    except Exception as e:
        _sr.info(f"EXCEPTION browsing source folder: {e}")
        import traceback
        _sr.info(traceback.format_exc())

@app.route('/api/playlists/<playlist_id>/download_missing', methods=['POST'])
def start_playlist_missing_downloads(playlist_id):
    """
    This endpoint receives the list of missing tracks and manages them with batch processing
    like the GUI, maintaining exactly 3 concurrent downloads.
    """
    data = request.get_json()
    missing_tracks = data.get('missing_tracks', [])
    if not missing_tracks:
        return jsonify({"success": False, "error": "No missing tracks provided"}), 400

    # Add activity for playlist download missing start
    playlist_name = data.get('playlist_name', f'Playlist {playlist_id}')
    add_activity_item("📥", "Missing Tracks Download Started", f"'{playlist_name}' - {len(missing_tracks)} tracks", "Now")

    try:
        batch_id = str(uuid.uuid4())
        
        # Create task queue for this batch
        task_queue = []
        with tasks_lock:
            # Initialize batch management
            download_batches[batch_id] = {
                'queue': [],
                'active_count': 0,
                'max_concurrent': 3,
                'queue_index': 0,
                # Track state management (replicating sync.py)
                'permanently_failed_tracks': [],
                'cancelled_tracks': set(),
                # Profile context for failed track wishlist re-adds
                'profile_id': get_current_profile_id()
            }

            for i, track_entry in enumerate(missing_tracks):
                task_id = str(uuid.uuid4())
                # Extract track data and original track index from frontend
                track_data = track_entry.get('track', track_entry)  # Support both old and new format
                original_track_index = track_entry.get('track_index', i)  # Use original index or fallback to enumeration
                
                download_tasks[task_id] = {
                    'status': 'pending',
                    'track_info': track_data,
                    'playlist_id': playlist_id,
                    'batch_id': batch_id,
                    'track_index': original_track_index,  # Use original playlist track index
                    'download_id': None,
                    'username': None,
                    'filename': None,
                    # Retry-related fields (GUI parity)
                    'retry_count': 0,
                    'cached_candidates': [],
                    'used_sources': set(),
                    'status_change_time': time.time()
                }
                
                # Add to batch queue instead of submitting immediately
                download_batches[batch_id]['queue'].append(task_id)
        
        # Start background monitoring for timeouts and retries (GUI parity)
        download_monitor.start_monitoring(batch_id)
        
        # Start the first batch of downloads (up to 3)
        _start_next_batch_of_downloads(batch_id)
        
        return jsonify({"success": True, "batch_id": batch_id, "message": f"Queued {len(missing_tracks)} downloads for processing."})
        
    except Exception as e:
        print(f"❌ Error starting missing downloads: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/active-processes', methods=['GET'])
def get_active_processes():
    """
    Returns all active processes for frontend rehydration:
    - Download batch processes (Spotify playlists)
    - YouTube discovery/sync processes (non-fresh phases)
    """
    active_processes = []
    
    # Add active download batch processes
    with tasks_lock:
        for batch_id, batch_data in download_batches.items():
            if batch_data.get('phase') not in ['complete', 'error', 'cancelled']:
                process_info = {
                    "type": "batch",
                    "playlist_id": batch_data.get('playlist_id'),
                    "playlist_name": batch_data.get('playlist_name'),
                    "batch_id": batch_id,
                    "phase": batch_data.get('phase')
                }
                
                # Enhanced wishlist information for better frontend state management
                if batch_data.get('playlist_id') == 'wishlist':
                    process_info.update({
                        "auto_initiated": batch_data.get('auto_initiated', False),
                        "auto_processing_timestamp": batch_data.get('auto_processing_timestamp'),
                        "should_show_modal": True,  # Wishlist processes should always be visible
                        "is_background_process": batch_data.get('auto_initiated', False),
                        "current_cycle": batch_data.get('current_cycle')  # Pass category filter to frontend
                    })

                    # Add current auto-processing state for frontend awareness
                    with wishlist_timer_lock:
                        process_info["auto_processing_active"] = wishlist_auto_processing
                
                active_processes.append(process_info)
    
    # Add YouTube playlists in non-fresh phases for rehydration
    for url_hash, state in youtube_playlist_states.items():
        # Include playlists that have progressed beyond fresh phase
        if state['phase'] != 'fresh':
            active_processes.append({
                "type": "youtube_playlist",
                "url_hash": url_hash,
                "url": state['url'],
                "playlist_name": state['playlist']['name'],
                "phase": state['phase'],
                "status": state['status'],
                "discovery_progress": state['discovery_progress'],
                "spotify_matches": state['spotify_matches'],
                "spotify_total": state['spotify_total'],
                "converted_spotify_playlist_id": state.get('converted_spotify_playlist_id'),
                "download_process_id": state.get('download_process_id')  # batch_id for download modal rehydration
            })
    
    print(f"📊 Active processes check: {len([p for p in active_processes if p['type'] == 'batch'])} download batches, {len([p for p in active_processes if p['type'] == 'youtube_playlist'])} YouTube playlists")
    return jsonify({"active_processes": active_processes})

def _build_batch_status_data(batch_id, batch, live_transfers_lookup):
    """
    Helper function to build status data for a single batch.
    Extracted from get_batch_download_status for reuse in batched endpoint.
    """
    response_data = {
        "phase": batch.get('phase', 'unknown'),
        "error": batch.get('error'),
        "auto_initiated": batch.get('auto_initiated', False),
        "playlist_id": batch.get('playlist_id'),  # Include playlist_id for rehydration
        "playlist_name": batch.get('playlist_name')  # Include playlist_name for reference
    }

    if response_data["phase"] == 'analysis':
        response_data['analysis_progress'] = {
            'total': batch.get('analysis_total', 0),
            'processed': batch.get('analysis_processed', 0)
        }
        response_data['analysis_results'] = batch.get('analysis_results', [])

    elif response_data["phase"] in ['downloading', 'complete', 'error']:
        response_data['analysis_results'] = batch.get('analysis_results', [])
        batch_tasks = []
        for task_id in batch.get('queue', []):
            task = download_tasks.get(task_id)
            if not task: continue

            # SAFETY VALVE: Check for downloads stuck too long
            import time
            current_time = time.time()
            task_start_time = task.get('status_change_time', current_time)
            task_age = current_time - task_start_time

            # If task has been running for more than 10 minutes, check if file completed
            if task_age > 600 and task['status'] in ['downloading', 'queued', 'searching']:
                stuck_state = task['status']
                task_filename = task.get('filename') or (task.get('track_info') or {}).get('filename')

                # Before failing, check if the file actually downloaded successfully
                recovered = False
                if task_filename and stuck_state == 'downloading':
                    try:
                        download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
                        transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer'))
                        found_file, file_location = _find_completed_file_robust(download_dir, task_filename, transfer_dir)
                        if found_file:
                            print(f"✅ [Safety Valve] Task {task_id} stuck but file found in {file_location} — routing to post-processing")
                            task['status'] = 'post_processing'
                            task['status_change_time'] = current_time
                            missing_download_executor.submit(_run_post_processing_worker, task_id, batch_id)
                            recovered = True
                    except Exception as e:
                        print(f"⚠️ [Safety Valve] Error checking for completed file: {e}")

                if not recovered:
                    if stuck_state == 'searching':
                        print(f"⏰ [Safety Valve] Task {task_id} stuck in searching for {task_age:.1f}s - marking not_found")
                        task['status'] = 'not_found'
                        task['error_message'] = f'Search stuck for {int(task_age // 60)} minutes with no results — timed out'
                    else:
                        print(f"⏰ [Safety Valve] Task {task_id} stuck for {task_age:.1f}s - forcing failure")
                        task['status'] = 'failed'
                        task['error_message'] = f'Task stuck in {stuck_state} state for {int(task_age // 60)} minutes — forcibly stopped'

            task_status = {
                'task_id': task_id,
                'track_index': task['track_index'],
                'status': task['status'],
                'track_info': task['track_info'],
                'progress': 0,
                # V2 SYSTEM: Add persistent state information
                'cancel_requested': task.get('cancel_requested', False),
                'cancel_timestamp': task.get('cancel_timestamp'),
                'ui_state': task.get('ui_state', 'normal'),  # normal|cancelling|cancelled
                'playlist_id': task.get('playlist_id'),      # For V2 system identification
                'error_message': task.get('error_message'),  # Surface failure reasons to UI
                'has_candidates': bool(task.get('cached_candidates')),  # Whether search found results (for clickable review)
            }
            _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {}
            task_filename = task.get('filename') or _ti.get('filename')
            task_username = task.get('username') or _ti.get('username')
            if task_filename and task_username:
                lookup_key = _make_context_key(task_username, task_filename)

                if lookup_key in live_transfers_lookup:
                    live_info = live_transfers_lookup[lookup_key]
                    state_str = live_info.get('state', 'Unknown')
                    
                    # Don't override tasks that are already in terminal states or post-processing
                    if task['status'] not in ['completed', 'failed', 'cancelled', 'not_found', 'post_processing']:
                        # SYNC.PY PARITY: Prioritized state checking (Errored/Cancelled before Completed)
                        # This prevents "Completed, Errored" states from being marked as completed
                        if 'Cancelled' in state_str or 'Canceled' in state_str:
                            task_status['status'] = 'cancelled'
                            task['status'] = 'cancelled'
                        elif 'Failed' in state_str or 'Errored' in state_str or 'Rejected' in state_str or 'TimedOut' in state_str:
                            # UNIFIED ERROR HANDLING: Let monitor handle errors for consistency
                            # Monitor will detect errored state and trigger retry within 5 seconds
                            print(f"🔍 Task {task_id} API shows error state: {state_str} - letting monitor handle retry")
                            
                            # Keep task in current status (downloading/queued) so monitor can detect error
                            # Don't mark as failed here - let the unified retry system handle it
                            if task['status'] in ['searching', 'downloading', 'queued']:
                                task_status['status'] = task['status']  # Keep current status for monitor
                            else:
                                task_status['status'] = 'downloading'  # Default to downloading for error detection
                                task['status'] = 'downloading'
                        elif 'Completed' in state_str or 'Succeeded' in state_str:
                            # Verify bytes actually transferred before trusting state string
                            expected_size = live_info.get('size', 0)
                            transferred = live_info.get('bytesTransferred', 0)
                            if expected_size > 0 and transferred < expected_size:
                                # State says complete but bytes don't match — keep current status
                                task_status['status'] = task['status']
                                print(f"⚠️ Task {task_id} state says complete but bytes incomplete ({transferred}/{expected_size})")
                            # NEW VERIFICATION WORKFLOW: Use intermediate post_processing status
                            # Only set this status once to prevent multiple worker submissions
                            elif task['status'] != 'post_processing':
                                task_status['status'] = 'post_processing'
                                task['status'] = 'post_processing'
                                print(f"🔄 Task {task_id} API reports 'Succeeded' - starting post-processing verification")
                                
                                # Submit post-processing worker to verify file and complete the task
                                missing_download_executor.submit(_run_post_processing_worker, task_id, batch_id)
                            else:
                                # FIXED: Always require verification workflow - no bypass for stream processed tasks
                                # Stream processing only handles metadata, not file verification
                                task_status['status'] = 'post_processing'
                                print(f"🔄 Task {task_id} waiting for verification worker to complete")
                        elif 'InProgress' in state_str: 
                            task_status['status'] = 'downloading'
                        else: 
                            task_status['status'] = 'queued'
                        task_status['progress'] = live_info.get('percentComplete', 0)
                    # For completed/post-processing tasks, keep appropriate progress
                    elif task['status'] == 'completed':
                        task_status['progress'] = 100
                    elif task['status'] == 'post_processing':
                        task_status['progress'] = 95  # Nearly complete, just verifying
                else:
                    # If task is completed but not in live transfers, keep appropriate status
                    if task['status'] == 'completed':
                        task_status['progress'] = 100
                    elif task['status'] == 'post_processing':
                        task_status['progress'] = 95  # Nearly complete, just verifying
            batch_tasks.append(task_status)
        batch_tasks.sort(key=lambda x: x['track_index'])
        response_data['tasks'] = batch_tasks
        
        # CRITICAL: Add batch worker management metadata (was missing!)
        # This is essential for client-side worker validation and prevents false desync warnings
        response_data['active_count'] = batch.get('active_count', 0)
        response_data['max_concurrent'] = batch.get('max_concurrent', 3)
        
        # Add wishlist summary if batch is complete (matching sync.py behavior)
        if response_data["phase"] == 'complete' and 'wishlist_summary' in batch:
            response_data['wishlist_summary'] = batch['wishlist_summary']

    return response_data

@app.route('/api/playlists/<batch_id>/download_status', methods=['GET'])
def get_batch_download_status(batch_id):
    """
    Returns real-time status for a single batch.
    Now uses shared helper function for consistency with batched endpoint.
    """
    try:
        # Use cached transfer data to reduce API calls with multiple concurrent modals
        live_transfers_lookup = get_cached_transfer_data()

        with tasks_lock:
            if batch_id not in download_batches:
                return jsonify({"error": "Batch not found"}), 404

            batch = download_batches[batch_id]
            response_data = _build_batch_status_data(batch_id, batch, live_transfers_lookup)
            return jsonify(response_data)

    except Exception as e:
        import traceback
        traceback.print_exc()
        return jsonify({"error": str(e)}), 500

@app.route('/api/download_status/batch', methods=['GET'])
def get_batched_download_statuses():
    """
    NEW: Returns status for multiple download batches in a single request.
    Dramatically reduces API calls when multiple download modals are active.
    
    Query params:
    - batch_ids: Optional list of specific batch IDs to include
    - If no batch_ids provided, returns all active batches
    """
    try:
        # Get optional batch ID filtering from query params
        requested_batch_ids = request.args.getlist('batch_ids')
        
        # Use shared cached transfer data - single lookup for all batches
        live_transfers_lookup = get_cached_transfer_data()
        
        response = {"batches": {}}
        
        with tasks_lock:
            # Determine which batches to include
            if requested_batch_ids:
                # Filter to only requested batch IDs that exist
                target_batches = {
                    bid: batch for bid, batch in download_batches.items() 
                    if bid in requested_batch_ids
                }
            else:
                # Return all active batches
                target_batches = download_batches.copy()
            
            # Build status data for each batch using shared helper
            for batch_id, batch in target_batches.items():
                try:
                    response["batches"][batch_id] = _build_batch_status_data(
                        batch_id, batch, live_transfers_lookup
                    )
                except Exception as batch_error:
                    # Don't fail entire request if one batch has issues
                    print(f"❌ Error processing batch {batch_id}: {batch_error}")
                    response["batches"][batch_id] = {"error": str(batch_error)}
        
        # Add metadata for debugging/monitoring
        response["metadata"] = {
            "total_batches": len(response["batches"]),
            "requested_batch_ids": requested_batch_ids,
            "timestamp": time.time()
        }
        
        # ENHANCED: Add comprehensive debug info for worker tracking
        debug_info = {}
        for batch_id, batch_status in response["batches"].items():
            if "error" not in batch_status:
                active_count = batch_status.get("active_count", 0)
                max_concurrent = batch_status.get("max_concurrent", 3)
                task_count = len(batch_status.get("tasks", []))
                active_tasks = len([t for t in batch_status.get("tasks", []) if t.get("status") in ['searching', 'downloading', 'queued']])
                
                debug_info[batch_id] = {
                    "reported_active": active_count,
                    "actual_active_tasks": active_tasks,
                    "max_concurrent": max_concurrent,
                    "total_tasks": task_count,
                    "worker_discrepancy": active_count != active_tasks
                }
        
        response["debug_info"] = debug_info
        
        print(f"📊 [Batched Status] Returning status for {len(response['batches'])} batches")
        
        # Log worker discrepancies for debugging
        discrepancies = [bid for bid, info in debug_info.items() if info.get("worker_discrepancy")]
        if discrepancies:
            print(f"⚠️ [Batched Status] Worker count discrepancies in batches: {discrepancies}")
        
        return jsonify(response)

    except Exception as e:
        import traceback
        traceback.print_exc()
        return jsonify({"error": str(e)}), 500

@app.route('/api/downloads/cancel_task', methods=['POST'])
def cancel_download_task():
    """
    Cancels a single, specific download task.
    This version is now identical to the GUI, adding the cancelled track to
    the wishlist for future automatic retries.
    """
    data = request.get_json()
    task_id = data.get('task_id')
    if not task_id:
        return jsonify({"success": False, "error": "Missing task_id"}), 400

    try:
        with tasks_lock:
            if task_id not in download_tasks:
                return jsonify({"success": False, "error": "Task not found"}), 404
            
            task = download_tasks[task_id]
            
            # Log current task state for debugging
            current_status = task.get('status', 'unknown')
            download_id = task.get('download_id')
            username = task.get('username')
            print(f"🔍 [Cancel Debug] Task {task_id} - Current status: '{current_status}', download_id: {download_id}, username: {username}")
            
            # Immediately mark as cancelled to prevent race conditions
            task['status'] = 'cancelled'
            
        # IMPROVED WORKER SLOT MANAGEMENT: Use batch state validation instead of task status
        batch_id = task.get('batch_id')
        worker_slot_freed = False
        
        if batch_id:
            try:
                # Check if we need to free a worker slot by examining batch state
                with tasks_lock:
                    if batch_id in download_batches:
                        batch = download_batches[batch_id]
                        active_count = batch['active_count']
                        
                        # Free worker slot if there are active workers and task was actively running
                        # This is more reliable than checking task status which can be inconsistent
                        if active_count > 0 and current_status in ['pending', 'searching', 'downloading', 'queued']:
                            print(f"🔄 [Cancel] Task {task_id} (status: {current_status}) - freeing worker slot for batch {batch_id}")
                            print(f"🔄 [Cancel] Active count before: {active_count}")
                            
                            # Use the completion callback with error handling
                            _on_download_completed(batch_id, task_id, success=False)
                            worker_slot_freed = True
                            
                            # Verify slot was actually freed
                            new_active = download_batches[batch_id]['active_count']
                            print(f"🔄 [Cancel] Active count after: {new_active}")
                            
                        elif active_count == 0:
                            print(f"🚫 [Cancel] Task {task_id} - no active workers to free")
                        else:
                            print(f"🚫 [Cancel] Task {task_id} (status: {current_status}) - not actively running, no slot to free")
                    else:
                        print(f"🚫 [Cancel] Task {task_id} - batch {batch_id} not found")
                        
            except Exception as slot_error:
                print(f"❌ [Cancel] Error managing worker slot for {task_id}: {slot_error}")
                # Attempt emergency recovery if normal completion failed
                if not worker_slot_freed:
                    try:
                        print(f"🚨 [Cancel] Attempting emergency worker slot recovery")
                        _recover_worker_slot(batch_id, task_id)
                    except Exception as recovery_error:
                        print(f"💀 [Cancel] FATAL: Emergency recovery failed: {recovery_error}")
        else:
            print(f"🚫 [Cancel] Task {task_id} cancelled (no batch_id - likely already completed)")

        # Optionally try to cancel the Soulseek download (don't block worker progression)
        if download_id and username:
            try:
                # This is an async call, so we run it and wait
                run_async(soulseek_client.cancel_download(download_id, username, remove=True))
                print(f"✅ Successfully cancelled Soulseek download {download_id} for task {task_id}")
            except Exception as e:
                print(f"⚠️ Warning: Failed to cancel download on slskd, but worker already moved on. Error: {e}")

        ### NEW LOGIC START: Add cancelled track to wishlist ###
        try:
            from core.wishlist_service import get_wishlist_service
            wishlist_service = get_wishlist_service()
            
            # The task dictionary contains all the necessary info
            track_info = task.get('track_info', {})
            
            # The wishlist service expects a dictionary with specific keys
            # We need to properly format the artists to avoid nested structures
            artists_data = track_info.get('artists', [])
            formatted_artists = []
            
            for artist in artists_data:
                if isinstance(artist, str):
                    # Already a string, use as-is
                    formatted_artists.append({'name': artist})
                elif isinstance(artist, dict):
                    # Check if it's already in the correct format
                    if 'name' in artist and isinstance(artist['name'], str):
                        # Already properly formatted
                        formatted_artists.append(artist)
                    elif 'name' in artist and isinstance(artist['name'], dict) and 'name' in artist['name']:
                        # Nested structure, extract the inner name
                        formatted_artists.append({'name': artist['name']['name']})
                    else:
                        # Fallback: convert to string
                        formatted_artists.append({'name': str(artist)})
                else:
                    # Fallback for any other type
                    formatted_artists.append({'name': str(artist)})
            
            # Build album data - preserve all fields (including artists) for correct folder placement
            album_raw = track_info.get('album', {})
            if isinstance(album_raw, dict):
                album_data = dict(album_raw)  # Copy all fields including artists
                album_data.setdefault('name', 'Unknown Album')
                album_data.setdefault('album_type', track_info.get('album_type', 'album'))
            else:
                album_data = {
                    'name': str(album_raw) if album_raw else 'Unknown Album',
                    'album_type': track_info.get('album_type', 'album')
                }

            spotify_track_data = {
                'id': track_info.get('id'),
                'name': track_info.get('name'),
                'artists': formatted_artists,
                'album': album_data,
                'duration_ms': track_info.get('duration_ms')
            }
            
            source_context = {
                'playlist_name': task.get('playlist_name', 'Unknown Playlist'),
                'playlist_id': task.get('playlist_id'),
                'added_from': 'modal_cancellation'
            }

            # Add to wishlist, treating cancellation as a failure
            # Pass the spotify data directly instead of creating a fake Track object
            success = wishlist_service.add_spotify_track_to_wishlist(
                spotify_track_data=spotify_track_data,
                failure_reason="Download cancelled by user",
                source_type="playlist",
                source_context=source_context,
                profile_id=get_current_profile_id()
            )

            if success:
                print(f"✅ Added cancelled track '{track_info.get('name')}' to wishlist.")
            else:
                print(f"❌ Failed to add cancelled track '{track_info.get('name')}' to wishlist.")
        except Exception as e:
            print(f"❌ CRITICAL ERROR adding cancelled track to wishlist: {e}")
        ### NEW LOGIC END ###

        return jsonify({"success": True, "message": "Task cancelled and added to wishlist for retry."})

    except Exception as e:
        return jsonify({"success": False, "error": str(e)}), 500

# ===============================
# NEW ATOMIC CANCEL SYSTEM V2
# ===============================

def _find_task_by_playlist_track(playlist_id, track_index):
    """
    Find task_id by playlist_id and track_index.
    This enables the new v2 API to work without requiring task_id from frontend.
    """
    for task_id, task in download_tasks.items():
        if (task.get('playlist_id') == playlist_id and 
            task.get('track_index') == track_index):
            return task_id, task
    return None, None

def _atomic_cancel_task(playlist_id, track_index):
    """
    Atomically cancel a single task with proper worker slot management.
    This is the core of the new cancel system - everything in one transaction.
    Returns: (success: bool, message: str, task_info: dict)
    """
    try:
        # Find the task to cancel
        task_id, task = _find_task_by_playlist_track(playlist_id, track_index)
        if not task_id:
            return False, f"Task not found for playlist {playlist_id}, track {track_index}", None
            
        # Check if already cancelled
        if task.get('status') == 'cancelled':
            return False, "Task already cancelled", {'task_id': task_id, 'status': 'cancelled'}
            
        current_status = task.get('status', 'unknown')
        original_status = current_status  # Store original status before changing it
        batch_id = task.get('batch_id')
        
        print(f"🎯 [Atomic Cancel] Starting atomic cancel: playlist={playlist_id}, track={track_index}, task={task_id}, status={current_status}")
        
        # Mark task as cancelled immediately (within same lock context)
        task['status'] = 'cancelled'
        task['cancel_requested'] = True
        task['cancel_timestamp'] = __import__('time').time()
        task['ui_state'] = 'cancelled'
        
        # Ensure task has persistent identifiers for V2 system
        if 'playlist_id' not in task:
            task['playlist_id'] = playlist_id
        
        # Handle worker slot management
        worker_slot_freed = False
        if batch_id and batch_id in download_batches:
            batch = download_batches[batch_id]
            active_count = batch['active_count']
            
            # Free worker slot if task was consuming one
            # More precise check: only free if task was actually running
            if active_count > 0 and current_status in ['pending', 'searching', 'downloading', 'queued']:
                print(f"🔄 [Atomic Cancel] Freeing worker slot for {task_id} (was {current_status})")
                
                # CRITICAL: Direct worker slot management to prevent _on_download_completed race
                old_active = batch['active_count']
                batch['active_count'] = max(0, old_active - 1)  # Prevent negative counts
                worker_slot_freed = True
                
                print(f"🔄 [Atomic Cancel] Worker count: {old_active} → {batch['active_count']}")
                
                # Try to start next task if available (still within lock)
                if (batch['queue_index'] < len(batch['queue']) and 
                    batch['active_count'] < batch['max_concurrent']):
                    print(f"🚀 [Atomic Cancel] Starting next task in queue")
                    # Call the existing function to start next downloads
                    # Note: This will be called outside the lock to prevent deadlock
                else:
                    print(f"🚫 [Atomic Cancel] No next task to start (queue_index: {batch['queue_index']}/{len(batch['queue'])}, active: {batch['active_count']}/{batch['max_concurrent']})")
        
        # Build result info
        task_info = {
            'task_id': task_id,
            'status': 'cancelled',
            'original_status': original_status,  # Pass original status for slskd cancellation
            'track_name': task.get('track_info', {}).get('name', 'Unknown'),
            'playlist_id': playlist_id,
            'track_index': track_index,
            'worker_slot_freed': worker_slot_freed
        }
        
        print(f"✅ [Atomic Cancel] Successfully cancelled task {task_id}")
        return True, "Task cancelled successfully", task_info
        
    except Exception as e:
        print(f"❌ [Atomic Cancel] Error in atomic cancel: {e}")
        import traceback
        traceback.print_exc()
        return False, f"Internal error: {str(e)}", None

@app.route('/api/downloads/cancel_task_v2', methods=['POST'])
def cancel_task_v2():
    """
    NEW ATOMIC CANCEL SYSTEM V2
    
    Accepts playlist_id and track_index instead of task_id.
    Performs atomic cancellation with proper worker slot management.
    No race conditions, no dual state management.
    """
    data = request.get_json()
    playlist_id = data.get('playlist_id')
    track_index = data.get('track_index')
    
    if not playlist_id or track_index is None:
        return jsonify({
            "success": False, 
            "error": "Missing playlist_id or track_index"
        }), 400

    try:
        # Everything in one atomic operation within the lock
        with tasks_lock:
            success, message, task_info = _atomic_cancel_task(playlist_id, track_index)
            
        if not success:
            return jsonify({"success": False, "error": message}), 400
            
        # Handle post-cancel operations (outside the lock to prevent deadlock)
        task_id = task_info['task_id']
        task = download_tasks.get(task_id)
        
        # Try to start next batch of downloads (this may start new workers)
        if task and task.get('batch_id'):
            batch_id = task['batch_id']
            # Call existing function to manage batch progression
            try:
                _start_next_batch_of_downloads(batch_id)
            except Exception as e:
                print(f"⚠️ [Atomic Cancel] Warning: Could not start next downloads: {e}")
            
            # CRITICAL: Check for batch completion after V2 cancel
            # V2 system bypasses _on_download_completed, so we need to check completion manually
            try:
                _check_batch_completion_v2(batch_id)
            except Exception as e:
                print(f"⚠️ [Atomic Cancel] Warning: Could not check batch completion: {e}")
        
        # Cancel Soulseek download if active (non-blocking)
        if task:
            download_id = task.get('download_id')
            username = task.get('username')
            current_status = task.get('status')
            original_status = task_info.get('original_status', current_status)  # Get original status from task_info
            
            print(f"🔍 [Atomic Cancel] Task {task_id} state: status='{current_status}', original_status='{original_status}', download_id='{download_id}', username='{username}'")
            print(f"🔍 [Atomic Cancel] Download ID type: {type(download_id)}, length: {len(str(download_id)) if download_id else 0}")
            backslash = '\\'
            print(f"🔍 [Atomic Cancel] Download ID looks like filename: {download_id and ('/' in str(download_id) or backslash in str(download_id))}")
            
            if download_id and username:
                # Always try to cancel in slskd - doesn't matter what status it was
                # If it's not there or already done, the DELETE request will just fail harmlessly
                try:
                        print(f"🚫 [Atomic Cancel] Attempting to cancel Soulseek download:")
                        print(f"   Username: {username}")  
                        print(f"   Download ID: {download_id}")
                        print(f"   Base URL: {soulseek_client.base_url}")
                        print(f"   Expected URL: {soulseek_client.base_url}/transfers/downloads/{username}/{download_id}?remove=true")
                        
                        # CRITICAL: Must use REAL download ID from slskd, not filename
                        success = False
                        real_download_id = None
                        
                        # Step 1: Always search for real download ID first
                        print(f"🔍 [Atomic Cancel] Searching slskd transfers for real download ID")
                        try:
                            all_transfers = run_async(soulseek_client._make_request('GET', 'transfers/downloads'))
                            if all_transfers:
                                # Look through transfers to find matching download
                                for user_data in all_transfers:
                                    if user_data.get('username') == username:
                                        for directory in user_data.get('directories', []):
                                            for file_data in directory.get('files', []):
                                                file_filename = file_data.get('filename', '')
                                                # Match by filename (our download_id might be filename)
                                                if (file_filename == download_id or 
                                                    __import__('os').path.basename(file_filename) == __import__('os').path.basename(str(download_id))):
                                                    real_download_id = file_data.get('id')
                                                    print(f"🎯 [Atomic Cancel] Found real download ID: {real_download_id} for file: {file_filename}")
                                                    break
                                            if real_download_id:
                                                break
                                    if real_download_id:
                                        break
                        except Exception as search_error:
                            print(f"⚠️ [Atomic Cancel] Error searching transfers: {search_error}")
                        
                        # Step 2: Try cancellation with real ID if found
                        if real_download_id:
                            print(f"🔄 [Atomic Cancel] Attempting cancel with real ID: {real_download_id}")
                            try:
                                # Use EXACT format from slskd web UI: DELETE /api/v0/transfers/downloads/{username}/{download_id}?remove=false
                                endpoint = f'transfers/downloads/{username}/{real_download_id}?remove=true'
                                print(f"🌐 [Atomic Cancel] Using slskd web UI format: {endpoint}")
                                
                                response = run_async(soulseek_client._make_request('DELETE', endpoint))
                                if response is not None:
                                    print(f"✅ [Atomic Cancel] Successfully cancelled with slskd web UI format: {real_download_id}")
                                    success = True
                                else:
                                    print(f"⚠️ [Atomic Cancel] Web UI format failed, trying alternative formats")
                                    
                                    # Fallback: Try without remove parameter
                                    endpoint2 = f'transfers/downloads/{username}/{real_download_id}'
                                    response2 = run_async(soulseek_client._make_request('DELETE', endpoint2))
                                    if response2 is not None:
                                        print(f"✅ [Atomic Cancel] Successfully cancelled without remove param: {real_download_id}")
                                        success = True
                                    else:
                                        # Final fallback: Try simple format (sync.py style)
                                        endpoint3 = f'transfers/downloads/{real_download_id}'
                                        response3 = run_async(soulseek_client._make_request('DELETE', endpoint3))
                                        if response3 is not None:
                                            print(f"✅ [Atomic Cancel] Successfully cancelled with simple format: {real_download_id}")
                                            success = True
                                        else:
                                            print(f"⚠️ [Atomic Cancel] All DELETE formats failed for real ID: {real_download_id}")
                            except Exception as cancel_error:
                                print(f"⚠️ [Atomic Cancel] Exception cancelling real ID {real_download_id}: {cancel_error}")
                        else:
                            print(f"⚠️ [Atomic Cancel] Could not find real download ID in slskd transfers")
                            print(f"🔄 [Atomic Cancel] This might be a pending download not yet in slskd - relying on status='cancelled' to prevent it")
                            # For pending downloads, the status='cancelled' will prevent them from starting
                            success = True  # Consider this success since pending downloads are prevented
                        
                        if not success:
                            print(f"❌ [Atomic Cancel] Failed to cancel download in slskd API")
                except Exception as e:
                    print(f"⚠️ [Atomic Cancel] Exception cancelling Soulseek download {download_id}: {e}")
                    # Print more details about the error
                    import traceback
                    print(f"⚠️ [Atomic Cancel] Cancel error traceback: {traceback.format_exc()}")
            else:
                print(f"ℹ️ [Atomic Cancel] No download_id or username available - skipping slskd cancel")
        
        # Add to wishlist (non-blocking, best effort)
        try:
            _add_cancelled_task_to_wishlist(task)
        except Exception as e:
            print(f"⚠️ [Atomic Cancel] Warning: Could not add to wishlist: {e}")
        
        return jsonify({
            "success": True,
            "message": message,
            "task_info": {
                'task_id': task_info['task_id'],
                'track_name': task_info['track_name'],
                'status': 'cancelled'
            }
        })
        
    except Exception as e:
        print(f"❌ [Cancel V2] Unexpected error: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

def _check_batch_completion_v2(batch_id):
    """
    V2 SYSTEM: Check if batch is complete after worker slot changes.
    
    This is needed because V2 atomic cancel bypasses _on_download_completed,
    so we need to manually check for batch completion.
    """
    try:
        with tasks_lock:
            if batch_id not in download_batches:
                print(f"⚠️ [Completion Check V2] Batch {batch_id} not found")
                return
            
            batch = download_batches[batch_id]
            all_tasks_started = batch['queue_index'] >= len(batch['queue'])
            no_active_workers = batch['active_count'] == 0
            
            # Count actually finished tasks (completed, failed, or cancelled)
            finished_count = 0
            retrying_count = 0
            queue = batch.get('queue', [])
            current_time = time.time()

            for task_id in queue:
                if task_id in download_tasks:
                    task = download_tasks[task_id]
                    task_status = task['status']

                    # STUCK DETECTION: Force fail tasks that have been in transitional states too long
                    if task_status == 'searching':
                        task_age = current_time - task.get('status_change_time', current_time)
                        if task_age > 600:  # 10 minutes
                            print(f"⏰ [Stuck Detection V2] Task {task_id} stuck in searching for {task_age:.0f}s - forcing not_found")
                            task['status'] = 'not_found'
                            task['error_message'] = f'Search stuck for {int(task_age // 60)} minutes with no results — timed out'
                            finished_count += 1
                        else:
                            retrying_count += 1
                    elif task_status == 'post_processing':
                        task_age = current_time - task.get('status_change_time', current_time)
                        if task_age > 300:  # 5 minutes (post-processing should be fast)
                            print(f"⏰ [Stuck Detection V2] Task {task_id} stuck in post_processing for {task_age:.0f}s - forcing completion")
                            task['status'] = 'completed'  # Assume it worked if file verification is taking too long
                            finished_count += 1
                        else:
                            retrying_count += 1
                    elif task_status in ['completed', 'failed', 'cancelled', 'not_found']:
                        finished_count += 1
                else:
                    # Task ID in queue but not in download_tasks - treat as completed to prevent blocking
                    print(f"⚠️ [Orphaned Task V2] Task {task_id} in queue but not in download_tasks - counting as finished")
                    finished_count += 1
            
            all_tasks_truly_finished = finished_count >= len(queue)
            has_retrying_tasks = retrying_count > 0
            
            print(f"🔍 [Completion Check V2] Batch {batch_id}: tasks_started={all_tasks_started}, workers={no_active_workers}, finished={finished_count}/{len(queue)}, retrying={retrying_count}")
            
            if all_tasks_started and no_active_workers and all_tasks_truly_finished and not has_retrying_tasks:
                # FIXED: Ensure batch is not already marked as complete to prevent duplicate processing
                if batch.get('phase') != 'complete':
                    print(f"🎉 [Completion Check V2] Batch {batch_id} is complete - marking as finished")

                    # Check if this is an auto-initiated batch
                    is_auto_batch = batch.get('auto_initiated', False)

                    # Mark batch as complete and set completion timestamp for auto-cleanup
                    batch['phase'] = 'complete'
                    batch['completion_time'] = time.time()  # Track when batch completed
                else:
                    print(f"✅ [Completion Check V2] Batch {batch_id} already marked complete - skipping duplicate processing")
                    return True  # Already complete
                
                # Update YouTube playlist phase to 'download_complete' if this is a YouTube playlist
                playlist_id = batch.get('playlist_id')
                if playlist_id and playlist_id.startswith('youtube_'):
                    url_hash = playlist_id.replace('youtube_', '')
                    if url_hash in youtube_playlist_states:
                        youtube_playlist_states[url_hash]['phase'] = 'download_complete'
                        print(f"📋 [Completion Check V2] Updated YouTube playlist {url_hash} to download_complete phase")
                
                # Update Tidal playlist phase to 'download_complete' if this is a Tidal playlist
                if playlist_id and playlist_id.startswith('tidal_'):
                    tidal_playlist_id = playlist_id.replace('tidal_', '')
                    if tidal_playlist_id in tidal_discovery_states:
                        tidal_discovery_states[tidal_playlist_id]['phase'] = 'download_complete'
                        print(f"📋 [Completion Check V2] Updated Tidal playlist {tidal_playlist_id} to download_complete phase")
                
                print(f"🎉 [Completion Check V2] Batch {batch_id} complete - stopping monitor")
                download_monitor.stop_monitoring(batch_id)

                # REPAIR: Scan all album folders from this batch for track number issues
                if repair_worker:
                    repair_worker.process_batch(batch_id)

        # Process wishlist outside of the lock to prevent threading issues
        if all_tasks_started and no_active_workers and all_tasks_truly_finished and not has_retrying_tasks:
            # Call wishlist processing outside the lock
            if is_auto_batch:
                print(f"🤖 [Completion Check V2] Processing auto-initiated batch completion")
                # Use the existing auto-completion function
                _process_failed_tracks_to_wishlist_exact_with_auto_completion(batch_id)
            else:
                print(f"📋 [Completion Check V2] Processing regular batch completion")
                # Use the regular completion function
                _process_failed_tracks_to_wishlist_exact(batch_id)
            
            return True  # Batch was completed
        else:
            print(f"📊 [Completion Check V2] Batch {batch_id} not yet complete: finished={finished_count}/{len(queue)}, retrying={retrying_count}, workers={batch['active_count']}")
            return False  # Batch still in progress
                
    except Exception as e:
        print(f"❌ [Completion Check V2] Error checking batch completion: {e}")
        import traceback
        traceback.print_exc()
        return False

def _add_cancelled_task_to_wishlist(task):
    """
    Helper function to add cancelled task to wishlist.
    Separated for clarity and error isolation.
    """
    if not task:
        return
        
    try:
        from core.wishlist_service import get_wishlist_service
        wishlist_service = get_wishlist_service()
        
        track_info = task.get('track_info', {})
        artists_data = track_info.get('artists', [])
        formatted_artists = []
        
        for artist in artists_data:
            if isinstance(artist, str):
                formatted_artists.append({'name': artist})
            elif isinstance(artist, dict):
                if 'name' in artist and isinstance(artist['name'], str):
                    formatted_artists.append(artist)
                elif 'name' in artist and isinstance(artist['name'], dict) and 'name' in artist['name']:
                    formatted_artists.append({'name': artist['name']['name']})
                else:
                    formatted_artists.append({'name': str(artist)})
            else:
                formatted_artists.append({'name': str(artist)})
        
        # Build album data - preserve all fields (including artists) for correct folder placement
        album_raw = track_info.get('album', {})
        if isinstance(album_raw, dict):
            album_data = dict(album_raw)  # Copy all fields including artists
            album_data.setdefault('name', 'Unknown Album')
            album_data.setdefault('album_type', track_info.get('album_type', 'album'))
            # Add images fallback if not present
            if 'images' not in album_data and track_info.get('album_image_url'):
                album_data['images'] = [{'url': track_info.get('album_image_url')}]
        else:
            # album is a string (album name)
            album_data = {
                'name': str(album_raw) if album_raw else 'Unknown Album',
                'album_type': track_info.get('album_type', 'album')
            }
            # Add album image if available
            if track_info.get('album_image_url'):
                album_data['images'] = [{'url': track_info.get('album_image_url')}]

        spotify_track_data = {
            'id': track_info.get('id'),
            'name': track_info.get('name'),
            'artists': formatted_artists,
            'album': album_data,
            'duration_ms': track_info.get('duration_ms')
        }

        source_context = {
            'playlist_name': task.get('playlist_name', 'Unknown Playlist'),
            'playlist_id': task.get('playlist_id'),
            'added_from': 'modal_cancellation_v2'
        }

        success = wishlist_service.add_spotify_track_to_wishlist(
            spotify_track_data=spotify_track_data,
            failure_reason="Download cancelled by user (v2)",
            source_type="playlist",
            source_context=source_context,
            profile_id=get_current_profile_id()
        )
        
        if success:
            print(f"✅ [Atomic Cancel] Added '{track_info.get('name')}' to wishlist")
        else:
            print(f"❌ [Atomic Cancel] Failed to add '{track_info.get('name')}' to wishlist")
            
    except Exception as e:
        print(f"❌ [Atomic Cancel] Critical error adding to wishlist: {e}")

@app.route('/api/playlists/<batch_id>/cancel_batch', methods=['POST'])
def cancel_batch(batch_id):
    """
    Cancels an entire batch - useful for cancelling during analysis phase 
    or cancelling all downloads at once.
    """
    try:
        with tasks_lock:
            if batch_id not in download_batches:
                return jsonify({"success": False, "error": "Batch not found"}), 404
            
            # Mark batch as cancelled
            download_batches[batch_id]['phase'] = 'cancelled'

            # Get playlist_id before doing resets
            playlist_id = download_batches[batch_id].get('playlist_id')

            # Reset wishlist auto-processing flag if this is a wishlist batch (auto-initiated only)
            # Manual wishlist downloads don't set the flag, so only reset if auto-initiated
            if playlist_id == 'wishlist':
                auto_initiated = download_batches[batch_id].get('auto_initiated', False)
                if auto_initiated:
                    global wishlist_auto_processing, wishlist_auto_processing_timestamp
                    with wishlist_timer_lock:
                        wishlist_auto_processing = False
                        wishlist_auto_processing_timestamp = 0
                    print(f"🔓 [Wishlist Cancel] Reset wishlist auto-processing flag for cancelled auto-batch")
                else:
                    print(f"ℹ️ [Wishlist Cancel] Manual wishlist batch cancelled (no flag reset needed)")

            # Reset YouTube playlist phase to 'discovered' if this is a YouTube playlist
            if playlist_id and playlist_id.startswith('youtube_'):
                url_hash = playlist_id.replace('youtube_', '')
                if url_hash in youtube_playlist_states:
                    youtube_playlist_states[url_hash]['phase'] = 'discovered'
                    print(f"📋 Reset YouTube playlist {url_hash} to discovered phase (batch cancelled)")
            
            # Cancel all individual tasks in the batch
            cancelled_count = 0
            for task_id in download_batches[batch_id].get('queue', []):
                if task_id in download_tasks:
                    task = download_tasks[task_id]
                    if task['status'] not in ['completed', 'failed', 'not_found', 'cancelled']:
                        task['status'] = 'cancelled'
                        cancelled_count += 1
            
            # Add activity for batch cancellation
            playlist_name = download_batches[batch_id].get('playlist_name', 'Unknown Playlist')
            add_activity_item("🚫", "Batch Cancelled", f"'{playlist_name}' - {cancelled_count} downloads cancelled", "Now")
            
            print(f"✅ Cancelled batch {batch_id} with {cancelled_count} tasks")
            return jsonify({"success": True, "cancelled_tasks": cancelled_count})
            
    except Exception as e:
        print(f"❌ Error cancelling batch {batch_id}: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

# NEW ENDPOINT: Add this function to web_server.py
@app.route('/api/playlists/cleanup_batch', methods=['POST'])
def cleanup_batch():
    """
    Cleans up a completed or cancelled batch from the server's in-memory state.
    This is called by the client after the user closes a finished modal.
    """
    data = request.get_json()
    batch_id = data.get('batch_id')
    if not batch_id:
        return jsonify({"success": False, "error": "Missing batch_id"}), 400

    try:
        with tasks_lock:
            # Check if the batch exists before trying to delete
            if batch_id in download_batches:
                batch = download_batches[batch_id]

                # CRITICAL: Don't allow cleanup if wishlist processing is in progress
                # This prevents a race condition where cleanup deletes the batch before
                # the wishlist processing thread can access it
                if batch.get('wishlist_processing_started') and not batch.get('wishlist_processing_complete'):
                    print(f"⏳ [Cleanup] Batch {batch_id} cleanup deferred - wishlist processing in progress")
                    return jsonify({
                        "success": False,
                        "error": "Batch cleanup deferred - wishlist processing in progress",
                        "deferred": True
                    }), 202  # 202 = Accepted but not yet processed

                # Get the list of task IDs before deleting the batch
                task_ids_to_remove = batch.get('queue', [])

                # Delete the batch record
                del download_batches[batch_id]
                
                # Clean up the associated tasks from the tasks dictionary
                for task_id in task_ids_to_remove:
                    if task_id in download_tasks:
                        del download_tasks[task_id]
                
                print(f"✅ Cleaned up batch '{batch_id}' and its associated tasks from server state.")
                return jsonify({"success": True, "message": f"Batch {batch_id} cleaned up."})
            else:
                # It's not an error if the batch is already gone
                print(f"⚠️ Cleanup requested for non-existent batch '{batch_id}'. Already cleaned up?")
                return jsonify({"success": True, "message": "Batch already cleaned up."})

    except Exception as e:
        print(f"❌ Error during batch cleanup for '{batch_id}': {e}")
        return jsonify({"success": False, "error": str(e)}), 500

# ===============================
# == UNIFIED MISSING TRACKS API ==
# ===============================

@app.route('/api/playlists/<playlist_id>/start-missing-process', methods=['POST'])
def start_missing_tracks_process(playlist_id):
    """
    A single, robust endpoint to kick off the entire missing tracks workflow.
    It creates a batch and starts the master worker in the background.
    """
    data = request.get_json()
    tracks = data.get('tracks', [])
    playlist_name = data.get('playlist_name', 'Unknown Playlist')
    force_download_all = data.get('force_download_all', False)
    playlist_folder_mode = data.get('playlist_folder_mode', False)

    # Get album/artist context for artist album downloads
    is_album_download = data.get('is_album_download', False)
    album_context = data.get('album_context', None)
    artist_context = data.get('artist_context', None)

    if not tracks:
        return jsonify({"success": False, "error": "No tracks provided"}), 400

    # Log album context if provided
    if is_album_download and album_context and artist_context:
        print(f"🎵 [Artist Album] Received album context: '{album_context.get('name')}' by '{artist_context.get('name')}' ({album_context.get('album_type', 'album')})")
        print(f"   Release: {album_context.get('release_date', 'Unknown')}, Tracks: {album_context.get('total_tracks', len(tracks))}")

    # Log playlist folder mode if enabled
    if playlist_folder_mode:
        print(f"📁 [Playlist Folder] Enabled for playlist: '{playlist_name}'")

    # Limit concurrent analysis processes to prevent resource exhaustion
    with tasks_lock:
        active_analysis_count = sum(1 for batch in download_batches.values() 
                                  if batch.get('phase') == 'analysis')
        if active_analysis_count >= 3:  # Allow max 3 concurrent analysis processes
            return jsonify({
                "success": False, 
                "error": "Too many analysis processes running. Please wait for one to complete."
            }), 429

    batch_id = str(uuid.uuid4())

    with tasks_lock:
        download_batches[batch_id] = {
            'phase': 'analysis',
            'playlist_id': playlist_id,
            'playlist_name': playlist_name,
            'queue': [],
            'active_count': 0,
            'max_concurrent': 1 if is_album_download else 3,  # Album/EP: 1 worker for source reuse; Playlist: 3 workers
            # Track state management (replicating sync.py)
            'permanently_failed_tracks': [],
            'cancelled_tracks': set(),
            'queue_index': 0,
            'analysis_total': len(tracks),
            # Profile context for failed track wishlist re-adds
            'profile_id': get_current_profile_id(),
            'analysis_processed': 0,
            'analysis_results': [],
            'force_download_all': force_download_all,  # Pass the force flag to the batch
            'playlist_folder_mode': playlist_folder_mode,  # Organize downloads by playlist folder
            # Album context for artist album downloads (explicit folder structure)
            'is_album_download': is_album_download,
            'album_context': album_context,
            'artist_context': artist_context
        }

    # Link YouTube playlist to download process if this is a YouTube playlist
    if playlist_id.startswith('youtube_'):
        url_hash = playlist_id.replace('youtube_', '')
        if url_hash in youtube_playlist_states:
            youtube_playlist_states[url_hash]['download_process_id'] = batch_id
            youtube_playlist_states[url_hash]['phase'] = 'downloading'
            youtube_playlist_states[url_hash]['converted_spotify_playlist_id'] = playlist_id
            print(f"🔗 Linked YouTube playlist {url_hash} to download process {batch_id} (converted ID: {playlist_id})")
    
    # Link Tidal playlist to download process if this is a Tidal playlist
    if playlist_id.startswith('tidal_'):
        tidal_playlist_id = playlist_id.replace('tidal_', '')
        if tidal_playlist_id in tidal_discovery_states:
            tidal_discovery_states[tidal_playlist_id]['download_process_id'] = batch_id
            tidal_discovery_states[tidal_playlist_id]['phase'] = 'downloading'
            tidal_discovery_states[tidal_playlist_id]['converted_spotify_playlist_id'] = playlist_id
            print(f"🔗 Linked Tidal playlist {tidal_playlist_id} to download process {batch_id} (converted ID: {playlist_id})")

    missing_download_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, tracks)

    return jsonify({
        "success": True,
        "batch_id": batch_id
    })

@app.route('/api/tracks/download_missing', methods=['POST'])
def start_missing_downloads():
    """Legacy endpoint - redirect to new playlist-based endpoint"""
    data = request.get_json()
    missing_tracks = data.get('missing_tracks', [])
    
    if not missing_tracks:
        return jsonify({"success": False, "error": "No missing tracks provided"}), 400
    
    # Use a default playlist_id for legacy compatibility
    playlist_id = "legacy_modal"
    
    # Call the new endpoint logic directly
    try:
        batch_id = str(uuid.uuid4())
        
        # Create task queue for this batch
        task_queue = []
        with tasks_lock:
            # Initialize batch management
            download_batches[batch_id] = {
                'queue': [],
                'active_count': 0,
                'max_concurrent': 3,
                'queue_index': 0,
                # Track state management (replicating sync.py)
                'permanently_failed_tracks': [],
                'cancelled_tracks': set(),
                # Profile context for failed track wishlist re-adds
                'profile_id': get_current_profile_id()
            }

            for track_index, track_data in enumerate(missing_tracks):
                task_id = str(uuid.uuid4())
                download_tasks[task_id] = {
                    'status': 'pending',
                    'track_info': track_data,
                    'playlist_id': playlist_id,
                    'batch_id': batch_id,
                    'track_index': track_index,
                    'download_id': None,
                    'username': None
                }
                
                # Add to batch queue instead of submitting immediately
                download_batches[batch_id]['queue'].append(task_id)
        
        # Start the first batch of downloads (up to 3)
        _start_next_batch_of_downloads(batch_id)

        return jsonify({"success": True, "batch_id": batch_id, "message": f"Queued {len(missing_tracks)} downloads for processing."})
        
    except Exception as e:
        print(f"❌ Error starting missing downloads: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

# ===============================
# == SYNC PAGE API             ==
# ===============================

def _load_sync_status_file():
    """Load sync statuses from database."""
    try:
        database = get_database()
        raw = database.get_preference('sync_statuses')
        if raw:
            data = json.loads(raw)
            return data
        return {}
    except Exception as e:
        print(f"❌ Error loading sync status: {e}")
        return {}

def _save_sync_status_file(sync_statuses):
    """Save sync statuses to database."""
    try:
        database = get_database()
        database.set_preference('sync_statuses', json.dumps(sync_statuses))
    except Exception as e:
        print(f"❌ Error saving sync status: {e}")

def _update_and_save_sync_status(playlist_id, playlist_name, playlist_owner, snapshot_id):
    """Updates the sync status for a given playlist and saves to file (same logic as GUI)."""
    try:
        # Load existing sync statuses
        sync_statuses = _load_sync_status_file()
        
        # Update this playlist's sync status
        from datetime import datetime
        now = datetime.now()
        sync_statuses[playlist_id] = {
            'name': playlist_name,
            'owner': playlist_owner,
            'snapshot_id': snapshot_id,
            'last_synced': now.isoformat()
        }
        
        # Save to file
        _save_sync_status_file(sync_statuses)
        print(f"🔄 Updated sync status for playlist '{playlist_name}' (ID: {playlist_id})")
        
    except Exception as e:
        print(f"❌ Error updating sync status for {playlist_id}: {e}")

@app.route('/api/spotify/playlists', methods=['GET'])
def get_spotify_playlists():
    """Fetches all user playlists from Spotify and enriches them with local sync status."""
    if not spotify_client or not spotify_client.is_authenticated():
        return jsonify({"error": "Spotify not authenticated."}), 401
    try:
        playlists = spotify_client.get_user_playlists_metadata_only()
        sync_statuses = _load_sync_status_file()

        playlist_data = []

        # Add regular playlists first
        for p in playlists:
            status_info = sync_statuses.get(p.id, {})
            sync_status = "Never Synced"
            # Handle snapshot_id safely - may not exist in core Playlist class
            playlist_snapshot = getattr(p, 'snapshot_id', '')

            print(f"🔍 Processing playlist: {p.name} (ID: {p.id})")
            print(f"   - Playlist snapshot: '{playlist_snapshot}'")
            print(f"   - Status info: {status_info}")

            if 'last_synced' in status_info:
                stored_snapshot = status_info.get('snapshot_id')
                last_sync_time = datetime.fromisoformat(status_info['last_synced']).strftime('%b %d, %H:%M')
                print(f"   - Stored snapshot: '{stored_snapshot}'")
                print(f"   - Snapshots match: {playlist_snapshot == stored_snapshot}")

                if playlist_snapshot != stored_snapshot:
                    sync_status = f"Last Sync: {last_sync_time}"
                    print(f"   - Result: Needs Sync (showing: {sync_status})")
                else:
                    sync_status = f"Synced: {last_sync_time}"
                    print(f"   - Result: {sync_status}")
            else:
                print(f"   - No last_synced found - Never Synced")

            playlist_data.append({
                "id": p.id, "name": p.name, "owner": p.owner,
                "track_count": p.total_tracks,
                "image_url": getattr(p, 'image_url', None),
                "sync_status": sync_status,
                "snapshot_id": playlist_snapshot
            })

        # Add virtual "Liked Songs" playlist at the END (just count, no full fetch)
        try:
            liked_songs_count = spotify_client.get_saved_tracks_count()
            if liked_songs_count > 0:
                liked_songs_id = "spotify:liked-songs"
                status_info = sync_statuses.get(liked_songs_id, {})
                sync_status = "Never Synced"

                if 'last_synced' in status_info:
                    last_sync_time = datetime.fromisoformat(status_info['last_synced']).strftime('%b %d, %H:%M')
                    sync_status = f"Synced: {last_sync_time}"

                # Get user info for owner name
                user_info = spotify_client.get_user_info()
                owner_name = user_info.get('display_name', 'You') if user_info else 'You'

                # Add Liked Songs as LAST playlist
                playlist_data.append({
                    "id": liked_songs_id,
                    "name": "Liked Songs",
                    "owner": owner_name,
                    "track_count": liked_songs_count,
                    "image_url": None,  # Spotify doesn't provide image for Liked Songs
                    "sync_status": sync_status,
                    "snapshot_id": ""  # Liked Songs doesn't have a snapshot_id
                })
                print(f"🔍 Added virtual 'Liked Songs' playlist with {liked_songs_count} tracks (count only)")
        except Exception as liked_error:
            print(f"⚠️ Failed to add Liked Songs playlist: {liked_error}")
            # Don't fail the entire request if Liked Songs fails

        return jsonify(playlist_data)
    except Exception as e:
        return jsonify({"error": str(e)}), 500

@app.route('/api/spotify/playlist/<playlist_id>', methods=['GET'])
def get_playlist_tracks(playlist_id):
    """Fetches full track details for a specific playlist."""
    if not spotify_client or not spotify_client.is_authenticated():
        return jsonify({"error": "Spotify not authenticated."}), 401
    try:
        # Handle special "Liked Songs" virtual playlist
        if playlist_id == "spotify:liked-songs":
            user_info = spotify_client.get_user_info()
            owner_name = user_info.get('display_name', 'You') if user_info else 'You'

            # Fetch raw saved tracks with full album data
            tracks = []
            limit = 50
            offset = 0

            while True:
                results = spotify_client.sp.current_user_saved_tracks(limit=limit, offset=offset)

                if not results or 'items' not in results:
                    break

                for item in results['items']:
                    if item['track'] and item['track']['id']:
                        track_data = item['track']
                        tracks.append({
                            'id': track_data['id'],
                            'name': track_data['name'],
                            'artists': track_data['artists'],  # Full artist objects (matches Download Missing Tracks behavior)
                            'album': track_data['album'],  # Full album object
                            'duration_ms': track_data['duration_ms'],
                            'popularity': track_data.get('popularity', 0),
                            'spotify_track_id': track_data['id']
                        })

                if len(results['items']) < limit or not results.get('next'):
                    break

                offset += limit

            # Create virtual playlist dict for Liked Songs
            playlist_dict = {
                'id': 'spotify:liked-songs',
                'name': 'Liked Songs',
                'description': 'Your saved tracks on Spotify',
                'owner': owner_name,
                'public': False,
                'collaborative': False,
                'track_count': len(tracks),
                'image_url': None,
                'snapshot_id': '',
                'tracks': tracks
            }
            return jsonify(playlist_dict)

        # Handle regular playlists
        # Fetch raw playlist data to preserve full album objects
        playlist_data = spotify_client.sp.playlist(playlist_id)

        # Fetch all tracks with full album data
        tracks = []
        results = spotify_client._get_playlist_items_page(playlist_id, limit=100)

        while results:
            for item in results['items']:
                # Handle both old API ('track') and new Feb 2026 API ('item') field names
                track_data = item.get('track') or item.get('item')
                if track_data and track_data.get('id'):
                    tracks.append({
                        'id': track_data['id'],
                        'name': track_data['name'],
                        'artists': track_data['artists'],  # Full artist objects (matches Download Missing Tracks behavior)
                        'album': track_data['album'],  # Full album object with album_type, total_tracks, etc.
                        'duration_ms': track_data['duration_ms'],
                        'popularity': track_data.get('popularity', 0),
                        'spotify_track_id': track_data['id']  # Also include as spotify_track_id for consistency
                    })

            results = spotify_client.sp.next(results) if results['next'] else None

        # Convert playlist to dict
        playlist_dict = {
            'id': playlist_data['id'],
            'name': playlist_data['name'],
            'description': playlist_data.get('description', ''),
            'owner': playlist_data['owner']['display_name'],
            'public': playlist_data.get('public', False),
            'collaborative': playlist_data.get('collaborative', False),
            'track_count': len(tracks),
            'image_url': playlist_data['images'][0]['url'] if playlist_data.get('images') else None,
            'snapshot_id': playlist_data.get('snapshot_id', ''),
            'tracks': tracks
        }
        return jsonify(playlist_dict)
    except Exception as e:
        return jsonify({"error": str(e)}), 500

@app.route('/api/spotify/album/<album_id>', methods=['GET'])
def get_album_tracks(album_id):
    """Fetches full track details for a specific album."""
    use_hydrabase = _is_hydrabase_active()

    # Try Hydrabase first when active — look up by album soul_id
    if use_hydrabase:
        album_name = request.args.get('name', '')
        album_artist = request.args.get('artist', '')
        try:
            hydra_tracks = hydrabase_client.get_album_tracks(album_id, limit=50)
            if hydra_tracks:
                track_items = []
                for t in hydra_tracks:
                    artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else []
                    track_items.append({
                        'name': t.name,
                        'track_number': t.track_number or 0,
                        'disc_number': t.disc_number or 1,
                        'duration_ms': t.duration_ms,
                        'id': t.id,
                        'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list],
                        'uri': ''
                    })
                return jsonify({
                    'id': album_id,
                    'name': album_name or hydra_tracks[0].album or '',
                    'artists': [{'name': album_artist}] if album_artist else [],
                    'release_date': '',
                    'total_tracks': len(track_items),
                    'album_type': 'album',
                    'images': [],
                    'tracks': track_items
                })
        except Exception as e:
            logger.warning(f"Hydrabase album_tracks failed for '{album_id}', falling back to Spotify: {e}")

    if not spotify_client or not spotify_client.is_authenticated():
        return jsonify({"error": "Spotify not authenticated."}), 401
    try:
        album_data = spotify_client.get_album(album_id)
        if not album_data:
            return jsonify({"error": "Album not found"}), 404

        # Extract tracks from album data (Spotify format)
        tracks = album_data.get('tracks', {}).get('items', [])

        # If no tracks in album data (iTunes format), fetch them separately
        if not tracks:
            tracks_data = spotify_client.get_album_tracks(album_id)
            if tracks_data and 'items' in tracks_data:
                tracks = tracks_data['items']

        # Format response
        album_dict = {
            'id': album_data['id'],
            'name': album_data['name'],
            'artists': album_data.get('artists', []),
            'release_date': album_data.get('release_date', ''),
            'total_tracks': album_data.get('total_tracks', 0),
            'album_type': album_data.get('album_type', 'album'),
            'images': album_data.get('images', []),
            'tracks': tracks
        }
        return jsonify(album_dict)
    except Exception as e:
        logger.error(f"Error fetching album tracks: {e}")
        return jsonify({"error": str(e)}), 500


@app.route('/api/spotify/track/<track_id>', methods=['GET'])
def get_spotify_track(track_id):
    """Fetches full track details including album data for a specific track."""
    # Try Hydrabase first when active and track name provided
    if _is_hydrabase_active():
        track_name = request.args.get('name', '')
        track_artist = request.args.get('artist', '')
        if track_name:
            try:
                query = f"{track_artist} {track_name}".strip() if track_artist else track_name
                hydra_tracks = hydrabase_client.search_tracks(query, limit=1)
                if hydra_tracks:
                    t = hydra_tracks[0]
                    artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else []
                    return jsonify({
                        'id': t.id,
                        'name': t.name,
                        'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list],
                        'album': {'name': t.album, 'images': [{'url': t.image_url}] if t.image_url else []},
                        'duration_ms': t.duration_ms,
                        'preview_url': t.preview_url,
                        'external_urls': t.external_urls or {},
                        'popularity': t.popularity,
                    })
            except Exception as e:
                logger.warning(f"Hydrabase track lookup failed for '{track_name}', falling back to Spotify: {e}")

    if not spotify_client or not spotify_client.is_authenticated():
        return jsonify({"error": "Spotify not authenticated."}), 401
    try:
        track_data = spotify_client.get_track_details(track_id)
        if not track_data:
            return jsonify({"error": "Track not found"}), 404

        return jsonify(track_data)
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route('/api/spotify/search', methods=['GET'])
def search_spotify():
    """Generic Spotify search endpoint - supports tracks, albums, artists"""
    use_hydrabase = _is_hydrabase_active()
    if not use_hydrabase:
        if not spotify_client or not spotify_client.is_authenticated():
            return jsonify({"error": "Spotify not authenticated."}), 401

    try:
        query = request.args.get('q', '').strip()
        search_type = request.args.get('type', 'track').strip()
        limit = int(request.args.get('limit', 20))

        if not query:
            return jsonify({"error": "Query parameter 'q' is required"}), 400

        if use_hydrabase:
            tracks = hydrabase_client.search_tracks(query, limit=limit)
        else:
            # Mirror to Hydrabase P2P network
            if hydrabase_worker and dev_mode_enabled:
                hydrabase_worker.enqueue(query, search_type)
            tracks = spotify_client.search_tracks(query, limit=limit)

        tracks_items = [{
            'id': t.id,
            'name': t.name,
            'artists': t.artists if isinstance(t.artists, list) else [t.artists],
            'album': t.album,
            'duration_ms': t.duration_ms,
            'uri': f"spotify:track:{t.id}"
        } for t in tracks]

        return jsonify({'tracks': {'items': tracks_items}})

    except Exception as e:
        print(f"❌ Error searching Spotify: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/spotify/search_tracks', methods=['GET'])
def search_spotify_tracks():
    """Search for tracks on Spotify - used by discovery fix modal"""
    use_hydrabase = _is_hydrabase_active()
    if not use_hydrabase:
        if not spotify_client or not spotify_client.is_authenticated():
            return jsonify({"error": "Spotify not authenticated."}), 401

    try:
        query = request.args.get('query', '').strip()
        limit = int(request.args.get('limit', 20))

        if not query:
            return jsonify({"error": "Query parameter is required"}), 400

        if use_hydrabase:
            tracks = hydrabase_client.search_tracks(query, limit=limit)
        else:
            if hydrabase_worker and dev_mode_enabled:
                hydrabase_worker.enqueue(query, 'track')
            tracks = spotify_client.search_tracks(query, limit=limit)

        tracks_dict = [{
            'id': t.id,
            'name': t.name,
            'artists': t.artists,
            'album': t.album,
            'duration_ms': t.duration_ms
        } for t in tracks]

        return jsonify({'tracks': tracks_dict})

    except Exception as e:
        print(f"❌ Error searching Spotify tracks: {e}")
        return jsonify({"error": str(e)}), 500


@app.route('/api/itunes/search_tracks', methods=['GET'])
def search_itunes_tracks():
    """Search for tracks on iTunes - used by discovery fix modal when iTunes is the source"""
    try:
        query = request.args.get('query', '').strip()
        limit = int(request.args.get('limit', 20))

        if not query:
            return jsonify({"error": "Query parameter is required"}), 400

        use_hydrabase = _is_hydrabase_active()
        if use_hydrabase:
            tracks = hydrabase_client.search_tracks(query, limit=limit)
            source = 'hydrabase'
        else:
            if hydrabase_worker and dev_mode_enabled:
                hydrabase_worker.enqueue(query, 'track')
            from core.itunes_client import iTunesClient
            itunes_client = iTunesClient()
            tracks = itunes_client.search_tracks(query, limit=limit)
            source = 'itunes'

        tracks_dict = [{
            'id': t.id,
            'name': t.name,
            'artists': t.artists,
            'album': t.album,
            'duration_ms': t.duration_ms,
            'image_url': t.image_url,
            'source': source
        } for t in tracks]

        return jsonify({'tracks': tracks_dict})

    except Exception as e:
        print(f"❌ Error searching iTunes tracks: {e}")
        return jsonify({"error": str(e)}), 500


@app.route('/api/itunes/album/<album_id>', methods=['GET'])
def get_itunes_album_tracks(album_id):
    """Fetches full track details for a specific iTunes album."""
    try:
        # Try Hydrabase first when active — look up by album soul_id
        if _is_hydrabase_active():
            album_name = request.args.get('name', '')
            album_artist = request.args.get('artist', '')
            try:
                hydra_tracks = hydrabase_client.get_album_tracks(album_id, limit=50)
                if hydra_tracks:
                    track_items = []
                    for t in hydra_tracks:
                        artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else []
                        track_items.append({
                            'name': t.name,
                            'track_number': t.track_number or 0,
                            'disc_number': t.disc_number or 1,
                            'duration_ms': t.duration_ms,
                            'id': t.id,
                            'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list],
                            'uri': ''
                        })
                    return jsonify({
                        'id': album_id,
                        'name': album_name or hydra_tracks[0].album or '',
                        'artists': [{'name': album_artist}] if album_artist else [],
                        'release_date': '',
                        'total_tracks': len(track_items),
                        'album_type': 'album',
                        'images': [],
                        'tracks': track_items,
                        'source': 'hydrabase'
                    })
            except Exception as e:
                logger.warning(f"Hydrabase album_tracks failed for '{album_id}', falling back to iTunes: {e}")

        from core.itunes_client import iTunesClient

        itunes_client = iTunesClient()
        album_data = itunes_client.get_album(album_id)

        if not album_data:
            return jsonify({"error": "Album not found"}), 404

        # Get tracks for this album
        tracks_data = itunes_client.get_album_tracks(album_id)
        tracks = tracks_data.get('items', []) if tracks_data else []

        # Format response to match Spotify structure for frontend compatibility
        album_dict = {
            'id': album_data.get('id', album_id),
            'name': album_data.get('name', 'Unknown Album'),
            'artists': album_data.get('artists', []),
            'release_date': album_data.get('release_date', ''),
            'total_tracks': album_data.get('total_tracks', len(tracks)),
            'album_type': album_data.get('album_type', 'album'),
            'images': album_data.get('images', []),
            'tracks': tracks,
            'source': 'itunes'
        }
        return jsonify(album_dict)

    except Exception as e:
        logger.error(f"Error fetching iTunes album tracks: {e}")
        return jsonify({"error": str(e)}), 500


@app.route('/api/discover/album/<source>/<album_id>', methods=['GET'])
def get_discover_album(source, album_id):
    """
    Source-agnostic album endpoint for discover page.
    Fetches album from the appropriate source (spotify, itunes, or hydrabase when active).
    """
    try:
        # Try Hydrabase first when active — look up by album soul_id
        if _is_hydrabase_active():
            album_name = request.args.get('name', '')
            album_artist = request.args.get('artist', '')
            try:
                hydra_tracks = hydrabase_client.get_album_tracks(album_id, limit=50)
                if hydra_tracks:
                    track_items = []
                    for t in hydra_tracks:
                        artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else []
                        track_items.append({
                            'name': t.name,
                            'track_number': t.track_number or 0,
                            'disc_number': t.disc_number or 1,
                            'duration_ms': t.duration_ms,
                            'id': t.id,
                            'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list],
                            'uri': ''
                        })
                    return jsonify({
                        'id': album_id,
                        'name': album_name or hydra_tracks[0].album or '',
                        'artists': [{'name': album_artist}] if album_artist else [],
                        'release_date': '',
                        'total_tracks': len(track_items),
                        'album_type': 'album',
                        'images': [],
                        'tracks': track_items,
                        'source': 'hydrabase'
                    })
            except Exception as e:
                logger.warning(f"Hydrabase album_tracks failed for '{album_id}', falling back to {source}: {e}")

        if source == 'spotify':
            if not spotify_client or not spotify_client.is_authenticated():
                return jsonify({"error": "Spotify not authenticated."}), 401

            album_data = spotify_client.get_album(album_id)
            if not album_data:
                return jsonify({"error": "Album not found"}), 404

            tracks = album_data.get('tracks', {}).get('items', [])
            if not tracks:
                tracks_data = spotify_client.get_album_tracks(album_id)
                if tracks_data and 'items' in tracks_data:
                    tracks = tracks_data['items']

            return jsonify({
                'id': album_data['id'],
                'name': album_data['name'],
                'artists': album_data.get('artists', []),
                'release_date': album_data.get('release_date', ''),
                'total_tracks': album_data.get('total_tracks', 0),
                'album_type': album_data.get('album_type', 'album'),
                'images': album_data.get('images', []),
                'tracks': tracks,
                'source': 'spotify'
            })

        elif source == 'itunes':
            from core.itunes_client import iTunesClient
            itunes_client = iTunesClient()

            album_data = itunes_client.get_album(album_id)
            if not album_data:
                return jsonify({"error": "Album not found"}), 404

            tracks_data = itunes_client.get_album_tracks(album_id)
            tracks = tracks_data.get('items', []) if tracks_data else []

            return jsonify({
                'id': album_data.get('id', album_id),
                'name': album_data.get('name', 'Unknown Album'),
                'artists': album_data.get('artists', []),
                'release_date': album_data.get('release_date', ''),
                'total_tracks': album_data.get('total_tracks', len(tracks)),
                'album_type': album_data.get('album_type', 'album'),
                'images': album_data.get('images', []),
                'tracks': tracks,
                'source': 'itunes'
            })

        else:
            return jsonify({"error": f"Unknown source: {source}"}), 400

    except Exception as e:
        logger.error(f"Error fetching discover album: {e}")
        return jsonify({"error": str(e)}), 500


# ===================================================================
# TIDAL DOWNLOAD AUTH ENDPOINTS
# ===================================================================

@app.route('/api/tidal/download/auth/start', methods=['POST'])
def tidal_download_auth_start():
    """Start Tidal device-code OAuth flow for download client."""
    try:
        tidal_dl = soulseek_client.tidal
        result = tidal_dl.start_device_auth()
        if result:
            return jsonify({"success": True, **result})
        else:
            return jsonify({"error": "Failed to start Tidal auth. Is tidalapi installed?"}), 500
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route('/api/tidal/download/auth/check', methods=['GET'])
def tidal_download_auth_check():
    """Check status of Tidal device-code OAuth flow."""
    try:
        tidal_dl = soulseek_client.tidal
        result = tidal_dl.check_device_auth()
        return jsonify(result)
    except Exception as e:
        return jsonify({"status": "error", "message": str(e)}), 500


@app.route('/api/tidal/download/auth/status', methods=['GET'])
def tidal_download_auth_status():
    """Check if Tidal download client is authenticated."""
    try:
        tidal_dl = soulseek_client.tidal
        authenticated = tidal_dl.is_authenticated()
        return jsonify({"authenticated": authenticated})
    except Exception as e:
        return jsonify({"authenticated": False, "error": str(e)})


# ===================================================================
# TIDAL PLAYLIST API ENDPOINTS
# ===================================================================

@app.route('/api/tidal/playlists', methods=['GET'])
def get_tidal_playlists():
    """Fetches all user playlists from Tidal with full track data (like sync.py)."""
    if not tidal_client or not tidal_client.is_authenticated():
        return jsonify({"error": "Tidal not authenticated."}), 401
    try:
        # Use same method as sync.py - this already includes all track data
        playlists = tidal_client.get_user_playlists_metadata_only()
        
        playlist_data = []
        for p in playlists:
            # Get track count from actual tracks if available
            track_count = len(p.tracks) if hasattr(p, 'tracks') and p.tracks else 0
            
            playlist_dict = {
                "id": p.id, 
                "name": p.name, 
                "owner": getattr(p, 'owner', 'Unknown'),
                "track_count": track_count,
                "image_url": getattr(p, 'image_url', None),
                "description": getattr(p, 'description', ''),
                "tracks": []  # Add tracks data like sync.py
            }
            
            # Include full track data if available (like sync.py has)
            if hasattr(p, 'tracks') and p.tracks:
                playlist_dict['tracks'] = [{
                    'id': t.id,
                    'name': t.name, 
                    'artists': t.artists or [],
                    'album': getattr(t, 'album', 'Unknown Album'),
                    'duration_ms': getattr(t, 'duration_ms', 0),
                    'track_number': getattr(t, 'track_number', 0)
                } for t in p.tracks]
                
            playlist_data.append(playlist_dict)
            
        print(f"🎵 Loaded {len(playlist_data)} Tidal playlists with track data")
        return jsonify(playlist_data)
    except Exception as e:
        return jsonify({"error": str(e)}), 500

@app.route('/api/tidal/playlist/<playlist_id>', methods=['GET'])
def get_tidal_playlist_tracks(playlist_id):
    """Fetches full track details for a specific Tidal playlist (matches sync.py pattern)."""
    if not tidal_client or not tidal_client.is_authenticated():
        return jsonify({"error": "Tidal not authenticated."}), 401
    try:
        print(f"🎵 Getting full Tidal playlist with tracks for: {playlist_id}")
        
        # First check if this playlist exists in metadata list
        try:
            metadata_playlists = tidal_client.get_user_playlists_metadata_only()
            target_playlist = None
            for p in metadata_playlists:
                if p.id == playlist_id:
                    target_playlist = p
                    break
            
            if not target_playlist:
                print(f"❌ Playlist {playlist_id} not found in user's Tidal playlists")
                return jsonify({"error": "Playlist not found in your Tidal library"}), 404
                
            print(f"🎵 Found playlist in metadata: {target_playlist.name}")
        except Exception as e:
            print(f"❌ Error checking playlist metadata: {e}")
        
        # Use same method as sync.py: tidal_client.get_playlist(playlist_id)
        full_playlist = tidal_client.get_playlist(playlist_id)
        if not full_playlist:
            return jsonify({"error": "Unable to access this Tidal playlist. This may be due to privacy settings or Tidal API restrictions. Please try a different playlist."}), 403
            
        if not full_playlist.tracks:
            return jsonify({"error": "This playlist appears to have no tracks or they cannot be accessed"}), 403
        
        print(f"🎵 Loaded {len(full_playlist.tracks)} tracks from Tidal playlist: {full_playlist.name}")
        
        # Convert playlist to dict (matches sync.py structure)
        playlist_dict = {
            'id': full_playlist.id,
            'name': full_playlist.name,
            'description': getattr(full_playlist, 'description', ''),
            'owner': getattr(full_playlist, 'owner', 'Unknown'),
            'track_count': len(full_playlist.tracks),
            'image_url': getattr(full_playlist, 'image_url', None),
            'tracks': []
        }
        
        # Convert tracks to dict format (for discovery modal)
        playlist_dict['tracks'] = [{
            'id': t.id,
            'name': t.name, 
            'artists': t.artists or [],
            'album': getattr(t, 'album', 'Unknown Album'),
            'duration_ms': getattr(t, 'duration_ms', 0),
            'track_number': getattr(t, 'track_number', 0)
        } for t in full_playlist.tracks]
        
        return jsonify(playlist_dict)
    except Exception as e:
        print(f"❌ Error getting Tidal playlist tracks: {e}")
        return jsonify({"error": str(e)}), 500


# ===================================================================
# TIDAL DISCOVERY API ENDPOINTS
# ===================================================================

# Global state for Tidal playlist discovery management
tidal_discovery_states = {}  # Key: playlist_id, Value: discovery state
tidal_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="tidal_discovery")

@app.route('/api/tidal/discovery/start/<playlist_id>', methods=['POST'])
def start_tidal_discovery(playlist_id):
    """Start Spotify discovery process for a Tidal playlist"""
    try:
        # Get playlist data from the initial load
        if not tidal_client or not tidal_client.is_authenticated():
            return jsonify({"error": "Tidal not authenticated."}), 401
            
        # Get playlist from tidal client
        playlists = tidal_client.get_user_playlists_metadata_only()
        target_playlist = None
        for p in playlists:
            if p.id == playlist_id:
                target_playlist = p
                break
                
        if not target_playlist:
            return jsonify({"error": "Tidal playlist not found"}), 404
            
        if not target_playlist.tracks:
            return jsonify({"error": "Playlist has no tracks"}), 400
        
        # Initialize discovery state if it doesn't exist, or update existing state
        if playlist_id in tidal_discovery_states:
            existing_state = tidal_discovery_states[playlist_id]
            if existing_state['phase'] == 'discovering':
                return jsonify({"error": "Discovery already in progress"}), 400
            # Update existing state for discovery
            existing_state['phase'] = 'discovering'
            existing_state['status'] = 'discovering' 
            existing_state['last_accessed'] = time.time()
            state = existing_state
        else:
            # Create new state for first-time discovery
            state = {
                'playlist': target_playlist,
                'phase': 'discovering', # fresh -> discovering -> discovered -> syncing -> sync_complete -> downloading -> download_complete
                'status': 'discovering',
                'discovery_progress': 0,
                'spotify_matches': 0,
                'spotify_total': len(target_playlist.tracks),
                'discovery_results': [],
                'sync_playlist_id': None,
                'converted_spotify_playlist_id': None,
                'download_process_id': None,  # Track associated download missing tracks process
                'created_at': time.time(),
                'last_accessed': time.time(),
                'discovery_future': None,
                'sync_progress': {}
            }
            tidal_discovery_states[playlist_id] = state
        
        # Add activity for discovery start
        add_activity_item("🔍", "Tidal Discovery Started", f"'{target_playlist.name}' - {len(target_playlist.tracks)} tracks", "Now")
        
        # Start discovery worker
        future = tidal_discovery_executor.submit(_run_tidal_discovery_worker, playlist_id)
        state['discovery_future'] = future
        
        print(f"🔍 Started Spotify discovery for Tidal playlist: {target_playlist.name}")
        return jsonify({"success": True, "message": "Discovery started"})
        
    except Exception as e:
        print(f"❌ Error starting Tidal discovery: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/tidal/discovery/status/<playlist_id>', methods=['GET'])
def get_tidal_discovery_status(playlist_id):
    """Get real-time discovery status for a Tidal playlist"""
    try:
        if playlist_id not in tidal_discovery_states:
            return jsonify({"error": "Tidal discovery not found"}), 404
        
        state = tidal_discovery_states[playlist_id]
        state['last_accessed'] = time.time()  # Update access time
        
        response = {
            'phase': state['phase'],
            'status': state['status'],
            'progress': state['discovery_progress'],
            'spotify_matches': state['spotify_matches'],
            'spotify_total': state['spotify_total'],
            'results': state['discovery_results'],
            'complete': state['phase'] == 'discovered'
        }
        
        return jsonify(response)

    except Exception as e:
        print(f"❌ Error getting Tidal discovery status: {e}")
        return jsonify({"error": str(e)}), 500


@app.route('/api/tidal/discovery/update_match', methods=['POST'])
def update_tidal_discovery_match():
    """Update a Tidal discovery result with manually selected Spotify track"""
    try:
        data = request.get_json()
        identifier = data.get('identifier')  # playlist_id
        track_index = data.get('track_index')
        spotify_track = data.get('spotify_track')

        if not identifier or track_index is None or not spotify_track:
            return jsonify({'error': 'Missing required fields'}), 400

        # Get the state
        state = tidal_discovery_states.get(identifier)

        if not state:
            return jsonify({'error': 'Discovery state not found'}), 404

        if track_index >= len(state['discovery_results']):
            return jsonify({'error': 'Invalid track index'}), 400

        # Update the result
        result = state['discovery_results'][track_index]
        old_status = result.get('status')

        # Update with user-selected track
        result['status'] = '✅ Found'
        result['status_class'] = 'found'
        result['spotify_track'] = spotify_track['name']
        result['spotify_artist'] = ', '.join(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else spotify_track['artists']
        result['spotify_album'] = spotify_track['album']
        result['spotify_id'] = spotify_track['id']

        # Format duration (Tidal doesn't show duration in table, but store it anyway)
        duration_ms = spotify_track.get('duration_ms', 0)
        if duration_ms:
            minutes = duration_ms // 60000
            seconds = (duration_ms % 60000) // 1000
            result['duration'] = f"{minutes}:{seconds:02d}"
        else:
            result['duration'] = '0:00'

        # IMPORTANT: Also set spotify_data for sync/download compatibility
        result['spotify_data'] = {
            'id': spotify_track['id'],
            'name': spotify_track['name'],
            'artists': spotify_track['artists'],
            'album': spotify_track['album'],
            'duration_ms': spotify_track.get('duration_ms', 0)
        }

        result['manual_match'] = True  # Flag for tracking

        # Update match count if status changed from not found/error
        if old_status != 'found' and old_status != '✅ Found':
            state['spotify_matches'] = state.get('spotify_matches', 0) + 1

        print(f"✅ Manual match updated: tidal - {identifier} - track {track_index}")
        print(f"   → {result['spotify_artist']} - {result['spotify_track']}")

        return jsonify({'success': True, 'result': result})

    except Exception as e:
        print(f"❌ Error updating Tidal discovery match: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/tidal/playlists/states', methods=['GET'])
def get_tidal_playlist_states():
    """Get all stored Tidal playlist discovery states for frontend hydration (similar to YouTube playlists)"""
    try:
        states = []
        current_time = time.time()
        
        for playlist_id, state in tidal_discovery_states.items():
            # Update access time when requested
            state['last_accessed'] = current_time
            
            # Return essential data for card state recreation
            state_info = {
                'playlist_id': playlist_id,
                'phase': state['phase'],
                'status': state['status'],
                'discovery_progress': state['discovery_progress'],
                'spotify_matches': state['spotify_matches'],
                'spotify_total': state['spotify_total'],
                'discovery_results': state['discovery_results'],
                'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'),
                'download_process_id': state.get('download_process_id'),
                'last_accessed': state['last_accessed']
            }
            states.append(state_info)
        
        print(f"🎵 Returning {len(states)} stored Tidal playlist states for hydration")
        return jsonify({"states": states})
        
    except Exception as e:
        print(f"❌ Error getting Tidal playlist states: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/tidal/state/<playlist_id>', methods=['GET'])
def get_tidal_playlist_state(playlist_id):
    """Get specific Tidal playlist state (detailed version matching YouTube's state endpoint)"""
    try:
        if playlist_id not in tidal_discovery_states:
            return jsonify({"error": "Tidal playlist not found"}), 404
        
        state = tidal_discovery_states[playlist_id]
        state['last_accessed'] = time.time()
        
        # Return full state information (including results for modal hydration)
        response = {
            'playlist_id': playlist_id,
            'playlist': state['playlist'].__dict__ if hasattr(state['playlist'], '__dict__') else state['playlist'],
            'phase': state['phase'],
            'status': state['status'],
            'discovery_progress': state['discovery_progress'],
            'spotify_matches': state['spotify_matches'],
            'spotify_total': state['spotify_total'],
            'discovery_results': state['discovery_results'],
            'sync_playlist_id': state.get('sync_playlist_id'),
            'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'),
            'download_process_id': state.get('download_process_id'),
            'sync_progress': state.get('sync_progress', {}),
            'last_accessed': state['last_accessed']
        }
        
        return jsonify(response)
        
    except Exception as e:
        print(f"❌ Error getting Tidal playlist state: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/tidal/reset/<playlist_id>', methods=['POST'])
def reset_tidal_playlist(playlist_id):
    """Reset Tidal playlist to fresh phase (clear discovery/sync data)"""
    try:
        if playlist_id not in tidal_discovery_states:
            return jsonify({"error": "Tidal playlist not found"}), 404
        
        state = tidal_discovery_states[playlist_id]
        
        # Stop any active discovery
        if 'discovery_future' in state and state['discovery_future']:
            state['discovery_future'].cancel()
        
        # Reset state to fresh (preserve original playlist data)
        state['phase'] = 'fresh'
        state['status'] = 'fresh'
        state['discovery_results'] = []
        state['discovery_progress'] = 0
        state['spotify_matches'] = 0
        state['sync_playlist_id'] = None
        state['converted_spotify_playlist_id'] = None
        state['download_process_id'] = None
        state['sync_progress'] = {}
        state['discovery_future'] = None
        state['last_accessed'] = time.time()
        
        print(f"🔄 Reset Tidal playlist to fresh: {playlist_id}")
        return jsonify({"success": True, "message": "Playlist reset to fresh phase"})
        
    except Exception as e:
        print(f"❌ Error resetting Tidal playlist: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/tidal/delete/<playlist_id>', methods=['POST'])
def delete_tidal_playlist(playlist_id):
    """Delete Tidal playlist state completely"""
    try:
        if playlist_id not in tidal_discovery_states:
            return jsonify({"error": "Tidal playlist not found"}), 404
        
        state = tidal_discovery_states[playlist_id]
        
        # Stop any active discovery
        if 'discovery_future' in state and state['discovery_future']:
            state['discovery_future'].cancel()
        
        # Remove from state dictionary
        del tidal_discovery_states[playlist_id]
        
        print(f"🗑️ Deleted Tidal playlist state: {playlist_id}")
        return jsonify({"success": True, "message": "Playlist deleted"})
        
    except Exception as e:
        print(f"❌ Error deleting Tidal playlist: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/tidal/update_phase/<playlist_id>', methods=['POST'])
def update_tidal_playlist_phase(playlist_id):
    """Update Tidal playlist phase (used when modal closes to reset from download_complete to discovered)"""
    try:
        if playlist_id not in tidal_discovery_states:
            return jsonify({"error": "Tidal playlist not found"}), 404
        
        data = request.get_json()
        if not data or 'phase' not in data:
            return jsonify({"error": "Phase not provided"}), 400
        
        new_phase = data['phase']
        valid_phases = ['fresh', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete']
        
        if new_phase not in valid_phases:
            return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400
        
        state = tidal_discovery_states[playlist_id]
        old_phase = state.get('phase', 'unknown')
        state['phase'] = new_phase
        state['last_accessed'] = time.time()
        
        print(f"🔄 Updated Tidal playlist {playlist_id} phase: {old_phase} → {new_phase}")
        return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase})
        
    except Exception as e:
        print(f"❌ Error updating Tidal playlist phase: {e}")
        return jsonify({"error": str(e)}), 500


def _run_playlist_discovery_worker(playlists, automation_id=None):
    """Background worker that discovers Spotify/iTunes metadata for undiscovered
    mirrored playlist tracks. Stores results in extra_data for use by sync."""
    try:
        use_spotify = spotify_client and spotify_client.is_spotify_authenticated()
        discovery_source = 'spotify' if use_spotify else 'itunes'

        itunes_client_instance = None
        if not use_spotify:
            try:
                from core.itunes_client import iTunesClient
                itunes_client_instance = iTunesClient()
            except Exception:
                print("❌ Neither Spotify nor iTunes available for discovery")
                _update_automation_progress(automation_id, status='error', progress=100,
                                            phase='Error', log_line='Neither Spotify nor iTunes available',
                                            log_type='error')
                return

        total_discovered = 0
        total_failed = 0
        total_skipped = 0
        total_tracks = 0
        last_playlist_name = ''

        # Pre-compute grand total for progress tracking (skip spotify playlists)
        grand_total = 0
        db_init = get_database()
        for pl in playlists:
            if pl.get('source', '') != 'spotify':
                t = db_init.get_mirrored_playlist_tracks(pl['id'])
                if t:
                    grand_total += len(t)
        _update_automation_progress(automation_id, total=grand_total)

        for pl in playlists:
            pl_id = pl['id']
            pl_name = pl.get('name', '')
            last_playlist_name = pl_name
            source = pl.get('source', '')

            # Spotify playlists are auto-discovered during refresh
            if source == 'spotify':
                continue

            db = get_database()
            tracks = db.get_mirrored_playlist_tracks(pl_id)
            if not tracks:
                continue

            print(f"🔍 Starting discovery for playlist '{pl_name}' ({len(tracks)} tracks, using {discovery_source.upper()})")
            _update_automation_progress(automation_id, phase=f'Discovering: "{pl_name}"',
                                         log_line=f'Playlist "{pl_name}" — {len(tracks)} tracks ({discovery_source.upper()})', log_type='info')

            for i, track in enumerate(tracks):
                total_tracks += 1
                track_id = track['id']
                track_name = track.get('track_name', '')
                artist_name = track.get('artist_name', '')
                duration_ms = track.get('duration_ms', 0)

                # Check if already discovered
                existing_extra = {}
                if track.get('extra_data'):
                    try:
                        existing_extra = json.loads(track['extra_data']) if isinstance(track['extra_data'], str) else track['extra_data']
                    except (json.JSONDecodeError, TypeError):
                        pass

                if existing_extra.get('discovered'):
                    total_skipped += 1
                    _update_automation_progress(automation_id, log_line=f'Already discovered: {track_name}', log_type='skip')
                    continue

                # Step 1: Check discovery cache
                cache_key = _get_discovery_cache_key(track_name, artist_name)
                try:
                    cached_match = db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source)
                    if cached_match and _validate_discovery_cache_artist(artist_name, cached_match):
                        extra_data = {
                            'discovered': True,
                            'provider': discovery_source,
                            'confidence': cached_match.get('confidence', 0.85),
                            'matched_data': cached_match,
                        }
                        db.update_mirrored_track_extra_data(track_id, extra_data)
                        total_discovered += 1
                        print(f"⚡ CACHE [{i+1}/{len(tracks)}]: {track_name} → {cached_match.get('name', '?')}")
                        _update_automation_progress(automation_id,
                            progress=(total_tracks / max(1, grand_total)) * 100,
                            current_item=track_name,
                            log_line=f'{track_name} → {cached_match.get("name", "?")} (cache)', log_type='success')
                        continue
                except Exception:
                    pass

                # Step 2: Generate search queries
                try:
                    temp_track = type('TempTrack', (), {
                        'name': track_name,
                        'artists': [artist_name],
                        'album': None
                    })()
                    search_queries = matching_engine.generate_download_queries(temp_track)
                except Exception:
                    search_queries = [f"{artist_name} {track_name}", track_name]

                # Step 3: Search and score
                best_match = None
                best_confidence = 0.0
                min_confidence = 0.7

                for search_query in search_queries:
                    try:
                        if use_spotify:
                            results = spotify_client.search_tracks(search_query, limit=10)
                        else:
                            results = itunes_client_instance.search_tracks(search_query, limit=10)
                        if not results:
                            continue

                        match, confidence, _ = _discovery_score_candidates(
                            track_name, artist_name, duration_ms, results
                        )

                        if match and confidence > best_confidence:
                            best_confidence = confidence
                            best_match = match

                        if best_confidence >= 0.9:
                            break
                    except Exception:
                        continue

                # Extended search fallback
                if not best_match or best_confidence < min_confidence:
                    try:
                        query = f"{artist_name} {track_name}"
                        if use_spotify:
                            extended = spotify_client.search_tracks(query, limit=50)
                        else:
                            extended = itunes_client_instance.search_tracks(query, limit=50)
                        if extended:
                            match, confidence, _ = _discovery_score_candidates(
                                track_name, artist_name, duration_ms, extended
                            )
                            if match and confidence > best_confidence:
                                best_confidence = confidence
                                best_match = match
                    except Exception:
                        pass

                # Step 4: Store results
                if best_match and best_confidence >= min_confidence:
                    match_artists = best_match.artists if hasattr(best_match, 'artists') else []
                    matched_data = {
                        'id': best_match.id if hasattr(best_match, 'id') else '',
                        'name': best_match.name if hasattr(best_match, 'name') else '',
                        'artists': [{'name': a} if isinstance(a, str) else a for a in match_artists],
                        'album': best_match.album if hasattr(best_match, 'album') else '',
                        'duration_ms': best_match.duration_ms if hasattr(best_match, 'duration_ms') else 0,
                        'source': discovery_source,
                    }

                    extra_data = {
                        'discovered': True,
                        'provider': discovery_source,
                        'confidence': best_confidence,
                        'matched_data': matched_data,
                    }
                    db.update_mirrored_track_extra_data(track_id, extra_data)
                    total_discovered += 1

                    # Save to discovery cache
                    try:
                        db.save_discovery_cache_match(
                            cache_key[0], cache_key[1], discovery_source,
                            best_confidence, matched_data,
                            track_name, artist_name
                        )
                    except Exception:
                        pass

                    print(f"✅ [{i+1}/{len(tracks)}] {track_name} → {matched_data['name']} ({best_confidence:.2f})")
                    _update_automation_progress(automation_id,
                        progress=(total_tracks / max(1, grand_total)) * 100,
                        processed=total_discovered + total_failed,
                        current_item=f'{track_name} - {artist_name}',
                        log_line=f'{track_name} → {matched_data["name"]} ({best_confidence:.2f})', log_type='success')
                else:
                    extra_data = {
                        'discovered': False,
                        'discovery_attempted': True,
                        'provider': discovery_source,
                    }
                    db.update_mirrored_track_extra_data(track_id, extra_data)
                    total_failed += 1
                    print(f"❌ [{i+1}/{len(tracks)}] No match: {track_name} by {artist_name}")
                    _update_automation_progress(automation_id,
                        progress=(total_tracks / max(1, grand_total)) * 100,
                        processed=total_discovered + total_failed,
                        current_item=f'{track_name} - {artist_name}',
                        log_line=f'{track_name} by {artist_name} → no match', log_type='error')

                time.sleep(0.15)

        # Emit completion event
        try:
            if automation_engine:
                automation_engine.emit('discovery_completed', {
                    'playlist_name': last_playlist_name if len(playlists) == 1 else f'{len(playlists)} playlists',
                    'total_tracks': str(total_tracks),
                    'discovered_count': str(total_discovered),
                    'failed_count': str(total_failed),
                    'skipped_count': str(total_skipped),
                })
        except Exception:
            pass

        print(f"✅ Playlist discovery complete: {total_discovered} discovered, {total_failed} failed, {total_skipped} skipped")
        _update_automation_progress(automation_id, status='finished', progress=100,
                                     phase='Discovery complete',
                                     log_line=f'Done: {total_discovered} discovered, {total_failed} failed, {total_skipped} skipped',
                                     log_type='success')

    except Exception as e:
        print(f"❌ Error in playlist discovery worker: {e}")
        import traceback
        traceback.print_exc()
        _update_automation_progress(automation_id, status='error', progress=100,
                                     phase='Error',
                                     log_line=f'Error: {str(e)}', log_type='error')


def _get_discovery_cache_key(title, artist):
    """Normalize title/artist for discovery cache lookup using matching_engine."""
    norm_title = matching_engine.clean_title(title)
    norm_artist = matching_engine.clean_artist(artist)
    return (norm_title, norm_artist)


def _validate_discovery_cache_artist(source_artist, cached_match):
    """Check if a cached discovery match has a valid artist. Returns False if the
    cached result's artist doesn't match the source artist (stale/wrong cache entry)."""
    min_artist_similarity = 0.5
    source_artist_cleaned = matching_engine.clean_artist(source_artist)
    if not source_artist_cleaned:
        return True  # No source artist to validate against

    cached_artists = cached_match.get('artists', [])
    if not cached_artists:
        return True  # No cached artist to check

    best_sim = 0.0
    for cand_artist in cached_artists:
        if not cand_artist:
            continue
        cand_normalized = matching_engine.normalize_string(cand_artist)
        if source_artist_cleaned in cand_normalized:
            return True
        cand_cleaned = matching_engine.clean_artist(cand_artist)
        sim = matching_engine.similarity_score(source_artist_cleaned, cand_cleaned)
        if sim > best_sim:
            best_sim = sim

    if best_sim < min_artist_similarity:
        print(f"🚫 Cache artist mismatch: source='{source_artist}' vs cached='{cached_artists[0]}' (sim={best_sim:.2f}), re-searching")
        return False
    return True


def _discovery_score_candidates(source_title, source_artist, source_duration_ms, search_results):
    """Score search results against a source track using the matching engine.

    Both artist AND title must independently pass minimum similarity floors.
    This prevents weighted scoring from allowing a perfect artist to carry a
    garbage title (or vice versa). If either dimension doesn't match, the
    candidate is rejected — no match is better than a wrong match.

    Args:
        source_title: The source track title (already cleaned for YouTube, raw for others)
        source_artist: The source track primary artist
        source_duration_ms: The source track duration in ms (0 if unknown)
        search_results: List of Track objects (Spotify or iTunes) from search

    Returns:
        (best_match, best_confidence, best_index) or (None, 0.0, -1) if no results
    """
    best_match = None
    best_confidence = 0.0
    best_index = -1
    min_artist_similarity = 0.5
    min_title_similarity = 0.5

    source_artist_cleaned = matching_engine.clean_artist(source_artist)
    source_title_cleaned = matching_engine.clean_title(source_title)
    source_core_title = matching_engine.get_core_string(source_title)

    for idx, result in enumerate(search_results):
        try:
            result_artists = result.artists if hasattr(result, 'artists') and result.artists else []
            result_name = result.name if hasattr(result, 'name') else ''
            result_duration = result.duration_ms if hasattr(result, 'duration_ms') else 0

            # Artist floor — both must match, not just the weighted score
            best_artist_sim = 0.0
            for cand_artist in result_artists:
                if not cand_artist:
                    continue
                cand_cleaned = matching_engine.clean_artist(cand_artist)
                cand_normalized = matching_engine.normalize_string(cand_artist)
                if source_artist_cleaned and source_artist_cleaned in cand_normalized:
                    best_artist_sim = 1.0
                    break
                sim = matching_engine.similarity_score(source_artist_cleaned, cand_cleaned)
                if sim > best_artist_sim:
                    best_artist_sim = sim

            if best_artist_sim < min_artist_similarity:
                continue

            # Title floor — both must match, not just the weighted score
            cand_title_cleaned = matching_engine.clean_title(result_name)
            cand_core_title = matching_engine.get_core_string(result_name)

            # Core title exact match bypasses the floor (e.g., "edamame" == "edamame")
            title_passes = False
            if source_core_title and cand_core_title and source_core_title == cand_core_title:
                title_passes = True
            else:
                title_sim = matching_engine.similarity_score(source_title_cleaned, cand_title_cleaned)
                if title_sim >= min_title_similarity:
                    title_passes = True

            if not title_passes:
                continue

            # Both floors passed — now do full scoring
            confidence, match_type = matching_engine.score_track_match(
                source_title=source_title,
                source_artists=[source_artist],
                source_duration_ms=source_duration_ms,
                candidate_title=result_name,
                candidate_artists=result_artists,
                candidate_duration_ms=result_duration
            )

            if confidence > best_confidence:
                best_confidence = confidence
                best_match = result
                best_index = idx

        except Exception as e:
            print(f"⚠️ Error scoring candidate {idx}: {e}")
            continue

    return best_match, best_confidence, best_index


def _run_tidal_discovery_worker(playlist_id):
    """Background worker for Tidal discovery process (Spotify preferred, iTunes fallback)"""
    try:
        state = tidal_discovery_states[playlist_id]
        playlist = state['playlist']

        # Determine which provider to use
        use_spotify = spotify_client and spotify_client.is_spotify_authenticated()
        discovery_source = 'spotify' if use_spotify else 'itunes'

        # Initialize iTunes client if needed
        itunes_client_instance = None
        if not use_spotify:
            from core.itunes_client import iTunesClient
            itunes_client_instance = iTunesClient()

        print(f"🎵 Starting Tidal discovery for: {playlist.name} (using {discovery_source.upper()})")

        # Store discovery source in state for frontend
        state['discovery_source'] = discovery_source

        successful_discoveries = 0

        for i, tidal_track in enumerate(playlist.tracks):
            if state.get('cancelled', False):
                break

            try:
                print(f"🔍 [{i+1}/{len(playlist.tracks)}] Searching {discovery_source.upper()}: {tidal_track.name} by {', '.join(tidal_track.artists)}")

                # Check discovery cache first
                cache_key = _get_discovery_cache_key(tidal_track.name, tidal_track.artists[0] if tidal_track.artists else '')
                try:
                    cache_db = get_database()
                    cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source)
                    if cached_match and _validate_discovery_cache_artist(tidal_track.artists[0] if tidal_track.artists else '', cached_match):
                        print(f"⚡ CACHE HIT [{i+1}/{len(playlist.tracks)}]: {tidal_track.name} by {', '.join(tidal_track.artists)}")
                        result = {
                            'tidal_track': {
                                'id': tidal_track.id,
                                'name': tidal_track.name,
                                'artists': tidal_track.artists or [],
                                'album': getattr(tidal_track, 'album', 'Unknown Album'),
                                'duration_ms': getattr(tidal_track, 'duration_ms', 0),
                            },
                            'spotify_data': cached_match,
                            'match_data': cached_match,
                            'status': 'found',
                            'discovery_source': discovery_source
                        }
                        successful_discoveries += 1
                        state['spotify_matches'] = successful_discoveries
                        state['discovery_results'].append(result)
                        state['discovery_progress'] = int(((i + 1) / len(playlist.tracks)) * 100)
                        continue
                except Exception as cache_err:
                    print(f"⚠️ Cache lookup error: {cache_err}")

                # Use the search function with appropriate provider
                track_result = _search_spotify_for_tidal_track(
                    tidal_track,
                    use_spotify=use_spotify,
                    itunes_client=itunes_client_instance
                )

                # Create result entry - use 'match_data' as generic key for both providers
                result = {
                    'tidal_track': {
                        'id': tidal_track.id,
                        'name': tidal_track.name,
                        'artists': tidal_track.artists or [],
                        'album': getattr(tidal_track, 'album', 'Unknown Album'),
                        'duration_ms': getattr(tidal_track, 'duration_ms', 0),
                    },
                    'spotify_data': None,  # Keep for backwards compatibility
                    'match_data': None,    # Generic field for any provider
                    'status': 'not_found',
                    'discovery_source': discovery_source
                }

                match_confidence = 0.0

                if use_spotify and isinstance(track_result, tuple):
                    # Spotify: Function returns (Track, raw_data, confidence)
                    track_obj, raw_track_data, match_confidence = track_result
                    album_obj = raw_track_data.get('album', {}) if raw_track_data else {}

                    match_data = {
                        'id': track_obj.id,
                        'name': track_obj.name,
                        'artists': track_obj.artists,
                        'album': album_obj,
                        'duration_ms': track_obj.duration_ms,
                        'external_urls': track_obj.external_urls,
                        'source': 'spotify'
                    }
                    result['spotify_data'] = match_data
                    result['match_data'] = match_data
                    result['status'] = 'found'
                    result['confidence'] = match_confidence
                    successful_discoveries += 1
                    state['spotify_matches'] = successful_discoveries

                elif not use_spotify and track_result and isinstance(track_result, dict):
                    # iTunes: Function returns a dict with track data (includes 'confidence' key)
                    match_confidence = track_result.pop('confidence', 0.80)
                    match_data = track_result
                    match_data['source'] = 'itunes'
                    result['spotify_data'] = match_data
                    result['match_data'] = match_data
                    result['status'] = 'found'
                    result['confidence'] = match_confidence
                    successful_discoveries += 1
                    state['spotify_matches'] = successful_discoveries

                # Save to discovery cache if match found
                if result['status'] == 'found' and result.get('match_data'):
                    try:
                        cache_db = get_database()
                        cache_db.save_discovery_cache_match(
                            cache_key[0], cache_key[1], discovery_source, match_confidence,
                            result['match_data'], tidal_track.name,
                            tidal_track.artists[0] if tidal_track.artists else ''
                        )
                        print(f"💾 CACHE SAVED: {tidal_track.name} (confidence: {match_confidence:.3f})")
                    except Exception as cache_err:
                        print(f"⚠️ Cache save error: {cache_err}")

                state['discovery_results'].append(result)
                state['discovery_progress'] = int(((i + 1) / len(playlist.tracks)) * 100)

                # Add delay between requests
                time.sleep(0.1)

            except Exception as e:
                print(f"❌ Error processing track {i+1}: {e}")
                # Add error result
                result = {
                    'tidal_track': {
                        'name': tidal_track.name,
                        'artists': tidal_track.artists or [],
                    },
                    'spotify_data': None,
                    'match_data': None,
                    'status': 'error',
                    'error': str(e),
                    'discovery_source': discovery_source
                }
                state['discovery_results'].append(result)
                state['discovery_progress'] = int(((i + 1) / len(playlist.tracks)) * 100)

        # Mark as complete
        state['phase'] = 'discovered'
        state['status'] = 'discovered'
        state['discovery_progress'] = 100

        # Add activity for discovery completion
        source_label = discovery_source.upper()
        add_activity_item("✅", f"Tidal Discovery Complete ({source_label})", f"'{playlist.name}' - {successful_discoveries}/{len(playlist.tracks)} tracks found", "Now")

        print(f"✅ Tidal discovery complete ({source_label}): {successful_discoveries}/{len(playlist.tracks)} tracks found")

    except Exception as e:
        print(f"❌ Error in Tidal discovery worker: {e}")
        state['phase'] = 'error'
        state['status'] = f'error: {str(e)}'


def _search_spotify_for_tidal_track(tidal_track, use_spotify=True, itunes_client=None):
    """Search Spotify/iTunes for a Tidal track using matching_engine for better accuracy

    Args:
        tidal_track: The Tidal track to search for
        use_spotify: If True, use Spotify; if False, use iTunes
        itunes_client: iTunes client instance (required when use_spotify=False)

    Returns:
        For Spotify: (Track, raw_data, confidence) tuple or None
        For iTunes: dict with track data (includes 'confidence' key) or None
    """
    if use_spotify:
        if not spotify_client or not spotify_client.is_authenticated():
            return None
    else:
        if not itunes_client:
            return None

    try:
        # Get track info
        track_name = tidal_track.name
        artists = tidal_track.artists or []

        if not artists:
            return None

        artist_name = artists[0]  # Use primary artist
        source_duration = getattr(tidal_track, 'duration_ms', 0) or 0
        source_name = "Spotify" if use_spotify else "iTunes"

        print(f"🔍 Tidal track: '{artist_name}' - '{track_name}' (searching {source_name})")

        # Use matching engine to generate search queries (with fallback)
        try:
            temp_track = type('TempTrack', (), {
                'name': track_name,
                'artists': [artist_name],
                'album': None
            })()
            search_queries = matching_engine.generate_download_queries(temp_track)
            print(f"🔍 Generated {len(search_queries)} search queries for Tidal track")
        except Exception as e:
            print(f"⚠️ Matching engine failed for Tidal, falling back to basic queries: {e}")
            if use_spotify:
                search_queries = [
                    f'track:"{track_name}" artist:"{artist_name}"',
                    f'"{track_name}" "{artist_name}"',
                    f'{track_name} {artist_name}'
                ]
            else:
                search_queries = [
                    f'{artist_name} {track_name}',
                    f'{track_name} {artist_name}',
                    track_name
                ]

        best_match = None
        best_match_raw = None
        best_confidence = 0.0
        min_confidence = 0.9

        for query_idx, search_query in enumerate(search_queries):
            try:
                print(f"🔍 Tidal query {query_idx + 1}/{len(search_queries)}: {search_query} ({source_name})")

                if use_spotify:
                    raw_results = spotify_client.sp.search(q=search_query, type='track', limit=10)
                    if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']:
                        continue
                    results = spotify_client.search_tracks(search_query, limit=10)
                    if not results:
                        continue
                else:
                    raw_results = None
                    results = itunes_client.search_tracks(search_query, limit=10)
                    if not results:
                        continue

                # Score all results using the matching engine
                match, confidence, match_idx = _discovery_score_candidates(
                    track_name, artist_name, source_duration, results
                )

                if match and confidence > best_confidence and confidence >= min_confidence:
                    best_confidence = confidence
                    best_match = match
                    if use_spotify and raw_results:
                        best_match_raw = raw_results['tracks']['items'][match_idx] if match_idx < len(raw_results['tracks']['items']) else None
                    print(f"✅ New best Tidal match: {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

                if best_confidence >= 0.9:
                    print(f"🎯 High confidence Tidal match found ({best_confidence:.3f}), stopping search")
                    break

            except Exception as e:
                print(f"❌ Error in Tidal {source_name} search for query '{search_query}': {e}")
                continue

        # Strategy 4: Extended search with higher limit (last resort)
        if not best_match:
            print(f"🔄 Tidal Strategy 4: Extended search with limit=50")
            query = f"{artist_name} {track_name}"
            if use_spotify:
                extended_results = spotify_client.search_tracks(query, limit=50)
            else:
                extended_results = itunes_client.search_tracks(query, limit=50)
            if extended_results:
                match, confidence, match_idx = _discovery_score_candidates(
                    track_name, artist_name, source_duration, extended_results
                )
                if match and confidence >= min_confidence:
                    best_match = match
                    best_confidence = confidence
                    print(f"✅ Strategy 4 Tidal match (extended): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

        if best_match:
            if use_spotify:
                print(f"✅ Final Tidal Spotify match: {best_match.artists[0]} - {best_match.name} (confidence: {best_confidence:.3f})")
                return (best_match, best_match_raw, best_confidence)
            else:
                result_artists = best_match.artists if hasattr(best_match, 'artists') else []
                result_artist = result_artists[0] if result_artists else 'Unknown'
                result_name = best_match.name if hasattr(best_match, 'name') else 'Unknown'
                print(f"✅ Final Tidal iTunes match: {result_artist} - {result_name} (confidence: {best_confidence:.3f})")

                album_name = best_match.album if hasattr(best_match, 'album') else 'Unknown Album'
                image_url = best_match.image_url if hasattr(best_match, 'image_url') else ''
                track_id = best_match.id if hasattr(best_match, 'id') else ''
                duration_ms = best_match.duration_ms if hasattr(best_match, 'duration_ms') else 0

                return {
                    'id': track_id,
                    'name': result_name,
                    'artists': [result_artist],
                    'album': {
                        'name': album_name,
                        'album_type': 'album',
                        'images': [{'url': image_url, 'height': 300, 'width': 300}] if image_url else []
                    },
                    'duration_ms': duration_ms,
                    'source': 'itunes',
                    'confidence': best_confidence
                }
        else:
            print(f"❌ No suitable Tidal match found (best confidence was {best_confidence:.3f}, required {min_confidence:.3f})")
            return None

    except Exception as e:
        print(f"❌ Error searching Spotify for Tidal track: {e}")
        return None


def convert_tidal_results_to_spotify_tracks(discovery_results):
    """Convert Tidal discovery results to Spotify tracks format for sync"""
    spotify_tracks = []

    for result in discovery_results:
        # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery)
        if result.get('spotify_data'):
            spotify_data = result['spotify_data']

            # Create track object matching the expected format
            track = {
                'id': spotify_data['id'],
                'name': spotify_data['name'],
                'artists': spotify_data['artists'],
                'album': spotify_data['album'],
                'duration_ms': spotify_data.get('duration_ms', 0)
            }
            spotify_tracks.append(track)
        elif result.get('spotify_track') and result.get('status_class') == 'found':
            # Build from individual fields (automatic discovery format)
            track = {
                'id': result.get('spotify_id', 'unknown'),
                'name': result.get('spotify_track', 'Unknown Track'),
                'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'],
                'album': result.get('spotify_album', 'Unknown Album'),
                'duration_ms': 0
            }
            spotify_tracks.append(track)

    print(f"🔄 Converted {len(spotify_tracks)} Tidal matches to Spotify tracks for sync")
    return spotify_tracks


# ===================================================================
# TIDAL SYNC API ENDPOINTS
# ===================================================================

@app.route('/api/tidal/sync/start/<playlist_id>', methods=['POST'])
def start_tidal_sync(playlist_id):
    """Start sync process for a Tidal playlist using discovered Spotify tracks"""
    try:
        if playlist_id not in tidal_discovery_states:
            return jsonify({"error": "Tidal playlist not found"}), 404
        
        state = tidal_discovery_states[playlist_id]
        state['last_accessed'] = time.time()  # Update access time
        
        if state['phase'] not in ['discovered', 'sync_complete']:
            return jsonify({"error": "Tidal playlist not ready for sync"}), 400
        
        # Convert discovery results to Spotify tracks format
        spotify_tracks = convert_tidal_results_to_spotify_tracks(state['discovery_results'])
        
        if not spotify_tracks:
            return jsonify({"error": "No Spotify matches found for sync"}), 400
        
        # Create a temporary playlist ID for sync tracking
        sync_playlist_id = f"tidal_{playlist_id}"
        playlist_name = state['playlist'].name  # Tidal playlist object has .name attribute
        
        # Add activity for sync start
        add_activity_item("🔄", "Tidal Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now")
        
        # Update Tidal state
        state['phase'] = 'syncing'
        state['sync_playlist_id'] = sync_playlist_id
        state['sync_progress'] = {}
        
        # Start the sync using existing sync infrastructure
        sync_data = {
            'playlist_id': sync_playlist_id,
            'playlist_name': playlist_name,
            'tracks': spotify_tracks
        }
        
        with sync_lock:
            sync_states[sync_playlist_id] = {"status": "starting", "progress": {}}
        
        # Submit sync task
        future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks)
        active_sync_workers[sync_playlist_id] = future
        
        print(f"🔄 Started Tidal sync for: {playlist_name} ({len(spotify_tracks)} tracks)")
        return jsonify({"success": True, "sync_playlist_id": sync_playlist_id})
        
    except Exception as e:
        print(f"❌ Error starting Tidal sync: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/tidal/sync/status/<playlist_id>', methods=['GET'])
def get_tidal_sync_status(playlist_id):
    """Get sync status for a Tidal playlist"""
    try:
        if playlist_id not in tidal_discovery_states:
            return jsonify({"error": "Tidal playlist not found"}), 404
        
        state = tidal_discovery_states[playlist_id]
        state['last_accessed'] = time.time()  # Update access time
        sync_playlist_id = state.get('sync_playlist_id')
        
        if not sync_playlist_id:
            return jsonify({"error": "No sync in progress"}), 404
        
        # Get sync status from existing sync infrastructure
        with sync_lock:
            sync_state = sync_states.get(sync_playlist_id, {})
        
        response = {
            'phase': state['phase'],
            'sync_status': sync_state.get('status', 'unknown'),
            'progress': sync_state.get('progress', {}),
            'complete': sync_state.get('status') == 'finished',
            'error': sync_state.get('error')
        }
        
        # Update Tidal state if sync completed
        if sync_state.get('status') == 'finished':
            state['phase'] = 'sync_complete'
            state['sync_progress'] = sync_state.get('progress', {})
            # Add activity for sync completion
            playlist = state.get('playlist')
            playlist_name = playlist.name if playlist and hasattr(playlist, 'name') else 'Unknown Playlist'
            add_activity_item("🔄", "Sync Complete", f"Tidal playlist '{playlist_name}' synced successfully", "Now")
        elif sync_state.get('status') == 'error':
            state['phase'] = 'discovered'  # Revert on error
            playlist = state.get('playlist')
            playlist_name = playlist.name if playlist and hasattr(playlist, 'name') else 'Unknown Playlist'
            add_activity_item("❌", "Sync Failed", f"Tidal playlist '{playlist_name}' sync failed", "Now")
        
        return jsonify(response)
        
    except Exception as e:
        print(f"❌ Error getting Tidal sync status: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/tidal/sync/cancel/<playlist_id>', methods=['POST'])
def cancel_tidal_sync(playlist_id):
    """Cancel sync for a Tidal playlist"""
    try:
        if playlist_id not in tidal_discovery_states:
            return jsonify({"error": "Tidal playlist not found"}), 404
        
        state = tidal_discovery_states[playlist_id]
        state['last_accessed'] = time.time()  # Update access time
        sync_playlist_id = state.get('sync_playlist_id')
        
        if sync_playlist_id:
            # Cancel the sync using existing sync infrastructure
            with sync_lock:
                sync_states[sync_playlist_id] = {"status": "cancelled"}
            
            # Clean up sync worker
            if sync_playlist_id in active_sync_workers:
                del active_sync_workers[sync_playlist_id]
        
        # Revert Tidal state
        state['phase'] = 'discovered'
        state['sync_playlist_id'] = None
        state['sync_progress'] = {}
        
        return jsonify({"success": True, "message": "Tidal sync cancelled"})
        
    except Exception as e:
        print(f"❌ Error cancelling Tidal sync: {e}")
        return jsonify({"error": str(e)}), 500


# ===================================================================
# YOUTUBE PLAYLIST API ENDPOINTS
# ===================================================================

# Global state for YouTube playlist management (persistent across page reloads)
youtube_playlist_states = {}  # Key: url_hash, Value: persistent playlist state
youtube_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="youtube_discovery")

# Global state for Beatport chart management (persistent across page reloads)
beatport_chart_states = {}  # Key: url_hash, Value: persistent chart state
beatport_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="beatport_discovery")

# Global state for ListenBrainz playlist management (persistent across page reloads)
listenbrainz_playlist_states = {}  # Key: playlist_mbid, Value: persistent playlist state
listenbrainz_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="listenbrainz_discovery")

@app.route('/api/youtube/parse', methods=['POST'])
def parse_youtube_playlist_endpoint():
    """Parse a YouTube playlist URL and return structured track data"""
    try:
        data = request.get_json()
        url = data.get('url', '').strip()
        
        if not url:
            return jsonify({"error": "YouTube URL is required"}), 400
        
        # Validate URL
        if not ('youtube.com/playlist' in url or 'music.youtube.com/playlist' in url):
            return jsonify({"error": "Invalid YouTube playlist URL"}), 400
        
        print(f"🎬 Parsing YouTube playlist: {url}")
        
        # Parse the playlist using our function
        playlist_data = parse_youtube_playlist(url)
        
        if not playlist_data:
            return jsonify({"error": "Failed to parse YouTube playlist"}), 500
        
        # Create URL hash for state tracking
        url_hash = str(hash(url))
        
        # Initialize persistent playlist state (similar to Spotify download_batches structure)
        youtube_playlist_states[url_hash] = {
            'playlist': playlist_data,
            'phase': 'fresh',  # fresh -> discovering -> discovered -> syncing -> sync_complete -> downloading -> download_complete
            'discovery_results': [],
            'discovery_progress': 0,
            'spotify_matches': 0,
            'spotify_total': len(playlist_data['tracks']),
            'status': 'parsed',
            'url': url,
            'sync_playlist_id': None,
            'converted_spotify_playlist_id': None,
            'download_process_id': None,  # Track associated download missing tracks process
            'created_at': time.time(),
            'last_accessed': time.time(),
            'discovery_future': None,
            'sync_progress': {}
        }
        
        playlist_data['url_hash'] = url_hash
        
        print(f"✅ YouTube playlist parsed successfully: {playlist_data['name']} ({len(playlist_data['tracks'])} tracks)")
        return jsonify(playlist_data)
        
    except Exception as e:
        print(f"❌ Error parsing YouTube playlist: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/youtube/discovery/start/<url_hash>', methods=['POST'])
def start_youtube_discovery(url_hash):
    """Start Spotify discovery process for a YouTube playlist"""
    try:
        if url_hash not in youtube_playlist_states:
            return jsonify({"error": "YouTube playlist not found"}), 404
        
        state = youtube_playlist_states[url_hash]
        state['last_accessed'] = time.time()  # Update access time
        
        if state['phase'] == 'discovering':
            return jsonify({"error": "Discovery already in progress"}), 400
        
        # Update phase to discovering
        state['phase'] = 'discovering'
        state['status'] = 'discovering'
        state['discovery_progress'] = 0
        state['spotify_matches'] = 0
        
        # Add activity for discovery start
        playlist_name = state['playlist']['name']
        track_count = len(state['playlist']['tracks'])
        add_activity_item("🔍", "YouTube Discovery Started", f"'{playlist_name}' - {track_count} tracks", "Now")
        
        # Start discovery worker
        future = youtube_discovery_executor.submit(_run_youtube_discovery_worker, url_hash)
        state['discovery_future'] = future
        
        print(f"🔍 Started Spotify discovery for YouTube playlist: {state['playlist']['name']}")
        return jsonify({"success": True, "message": "Discovery started"})
        
    except Exception as e:
        print(f"❌ Error starting YouTube discovery: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/youtube/discovery/status/<url_hash>', methods=['GET'])
def get_youtube_discovery_status(url_hash):
    """Get real-time discovery status for a YouTube playlist"""
    try:
        if url_hash not in youtube_playlist_states:
            return jsonify({"error": "YouTube playlist not found"}), 404
        
        state = youtube_playlist_states[url_hash]
        state['last_accessed'] = time.time()  # Update access time
        
        response = {
            'phase': state['phase'],
            'status': state['status'],
            'progress': state['discovery_progress'],
            'spotify_matches': state['spotify_matches'],
            'spotify_total': state['spotify_total'],
            'results': state['discovery_results'],
            'complete': state['phase'] == 'discovered'
        }
        
        return jsonify(response)

    except Exception as e:
        print(f"❌ Error getting YouTube discovery status: {e}")
        return jsonify({"error": str(e)}), 500


@app.route('/api/youtube/discovery/update_match', methods=['POST'])
def update_youtube_discovery_match():
    """Update a YouTube discovery result with manually selected Spotify track"""
    try:
        data = request.get_json()
        identifier = data.get('identifier')  # url_hash
        track_index = data.get('track_index')
        spotify_track = data.get('spotify_track')

        if not identifier or track_index is None or not spotify_track:
            return jsonify({'error': 'Missing required fields'}), 400

        # Get the state
        state = youtube_playlist_states.get(identifier)

        if not state:
            return jsonify({'error': 'Discovery state not found'}), 404

        if track_index >= len(state['discovery_results']):
            return jsonify({'error': 'Invalid track index'}), 400

        # Update the result
        result = state['discovery_results'][track_index]
        old_status = result.get('status')

        # Update with user-selected track
        result['status'] = '✅ Found'
        result['status_class'] = 'found'
        result['spotify_track'] = spotify_track['name']
        result['spotify_artist'] = ', '.join(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else spotify_track['artists']
        result['spotify_album'] = spotify_track['album']
        result['spotify_id'] = spotify_track['id']

        # Format duration
        duration_ms = spotify_track.get('duration_ms', 0)
        if duration_ms:
            minutes = duration_ms // 60000
            seconds = (duration_ms % 60000) // 1000
            result['duration'] = f"{minutes}:{seconds:02d}"
        else:
            result['duration'] = '0:00'

        # IMPORTANT: Also set spotify_data for sync/download compatibility
        result['spotify_data'] = {
            'id': spotify_track['id'],
            'name': spotify_track['name'],
            'artists': spotify_track['artists'],
            'album': spotify_track['album'],
            'duration_ms': spotify_track.get('duration_ms', 0)
        }

        result['manual_match'] = True  # Flag for tracking

        # Update match count if status changed from not found/error
        if old_status != 'found' and old_status != '✅ Found':
            state['spotify_matches'] = state.get('spotify_matches', 0) + 1

        print(f"✅ Manual match updated: youtube - {identifier} - track {track_index}")
        print(f"   → {result['spotify_artist']} - {result['spotify_track']}")

        return jsonify({'success': True, 'result': result})

    except Exception as e:
        print(f"❌ Error updating YouTube discovery match: {e}")
        return jsonify({'error': str(e)}), 500


def _run_youtube_discovery_worker(url_hash):
    """Background worker for YouTube music discovery process (Spotify preferred, iTunes fallback)"""
    try:
        state = youtube_playlist_states[url_hash]
        playlist = state['playlist']
        tracks = playlist['tracks']

        # Determine which provider to use (Spotify preferred, iTunes fallback)
        use_spotify = spotify_client and spotify_client.is_spotify_authenticated()
        discovery_source = 'spotify' if use_spotify else 'itunes'

        # Get iTunes client for fallback
        from core.itunes_client import iTunesClient
        itunes_client = iTunesClient()

        print(f"🔍 Starting {discovery_source} discovery for {len(tracks)} YouTube tracks...")

        # Store the discovery source in state
        state['discovery_source'] = discovery_source
        
        # Process each track for discovery
        for i, track in enumerate(tracks):
            try:
                # Update progress
                state['discovery_progress'] = int((i / len(tracks)) * 100)

                # Search for track using active provider
                cleaned_title = track['name']
                cleaned_artist = track['artists'][0] if track['artists'] else 'Unknown Artist'

                print(f"🔍 Searching {discovery_source} for: '{cleaned_artist}' - '{cleaned_title}'")

                # Check discovery cache first
                cache_key = _get_discovery_cache_key(cleaned_title, cleaned_artist)
                try:
                    cache_db = get_database()
                    cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source)
                    if cached_match and _validate_discovery_cache_artist(cleaned_artist, cached_match):
                        print(f"⚡ CACHE HIT [{i+1}/{len(tracks)}]: {cleaned_artist} - {cleaned_title}")
                        result = {
                            'index': i,
                            'yt_track': cleaned_title,
                            'yt_artist': cleaned_artist,
                            'status': '✅ Found',
                            'status_class': 'found',
                            'spotify_track': cached_match.get('name', ''),
                            'spotify_artist': cached_match.get('artists', [''])[0] if cached_match.get('artists') else '',
                            'spotify_album': cached_match.get('album', {}).get('name', '') if isinstance(cached_match.get('album'), dict) else cached_match.get('album', ''),
                            'duration': f"{track['duration_ms'] // 60000}:{(track['duration_ms'] % 60000) // 1000:02d}" if track['duration_ms'] else '0:00',
                            'discovery_source': discovery_source,
                            'matched_data': cached_match,
                            'spotify_data': cached_match
                        }
                        state['spotify_matches'] += 1
                        state['discovery_results'].append(result)
                        continue
                except Exception as cache_err:
                    print(f"⚠️ Cache lookup error: {cache_err}")

                # Try multiple search strategies using matching engine
                matched_track = None
                best_confidence = 0.0
                best_raw_track = None
                min_confidence = 0.9
                source_duration = track.get('duration_ms', 0) or 0

                # Strategy 1: Use matching_engine search queries
                try:
                    temp_track = type('TempTrack', (), {
                        'name': cleaned_title,
                        'artists': [cleaned_artist],
                        'album': None
                    })()
                    search_queries = matching_engine.generate_download_queries(temp_track)
                    print(f"🔍 Generated {len(search_queries)} search queries for YouTube track")
                except Exception as e:
                    print(f"⚠️ Matching engine failed for YouTube, falling back to basic query: {e}")
                    search_queries = [f"{cleaned_artist} {cleaned_title}", cleaned_title]

                for query_idx, search_query in enumerate(search_queries):
                    try:
                        print(f"🔍 YouTube query {query_idx + 1}/{len(search_queries)}: {search_query}")

                        raw_results = None
                        search_results = None

                        if use_spotify:
                            raw_results = spotify_client.sp.search(q=search_query, type='track', limit=10)
                            if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']:
                                continue
                            search_results = spotify_client.search_tracks(search_query, limit=10)
                        else:
                            search_results = itunes_client.search_tracks(search_query, limit=10)

                        if not search_results:
                            continue

                        # Score all results using the matching engine
                        match, confidence, match_idx = _discovery_score_candidates(
                            cleaned_title, cleaned_artist, source_duration, search_results
                        )

                        if match and confidence > best_confidence and confidence >= min_confidence:
                            best_confidence = confidence
                            matched_track = match
                            if use_spotify and raw_results:
                                best_raw_track = raw_results['tracks']['items'][match_idx] if match_idx < len(raw_results['tracks']['items']) else None
                            else:
                                best_raw_track = None
                            print(f"✅ New best YouTube match: {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

                        if best_confidence >= 0.9:
                            print(f"🎯 High confidence YouTube match found ({best_confidence:.3f}), stopping search")
                            break

                    except Exception as e:
                        print(f"❌ Error in YouTube search for query '{search_query}': {e}")
                        continue

                if matched_track:
                    print(f"✅ Strategy 1 YouTube match: {matched_track.artists[0]} - {matched_track.name} (confidence: {best_confidence:.3f})")

                # Strategy 2: Swapped search (if first failed) - score results properly
                if not matched_track:
                    print("🔄 YouTube Strategy 2: Trying swapped search (artist/title reversed)")
                    if use_spotify:
                        query = f"artist:{cleaned_title} track:{cleaned_artist}"
                        fallback_results = spotify_client.search_tracks(query, limit=5)
                    else:
                        query = f"{cleaned_title} {cleaned_artist}"
                        fallback_results = itunes_client.search_tracks(query, limit=5)
                    if fallback_results:
                        match, confidence, _ = _discovery_score_candidates(
                            cleaned_title, cleaned_artist, source_duration, fallback_results
                        )
                        if match and confidence >= min_confidence:
                            matched_track = match
                            best_confidence = confidence
                            print(f"✅ Strategy 2 YouTube match (swapped): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

                # Strategy 3: Raw data search (if still failed) - score results properly
                if not matched_track:
                    raw_title = track.get('raw_title', cleaned_title)
                    raw_artist = track.get('raw_artist', cleaned_artist)
                    print(f"🔄 YouTube Strategy 3: Trying raw data search: '{raw_artist} {raw_title}'")
                    query = f"{raw_artist} {raw_title}"
                    if use_spotify:
                        fallback_results = spotify_client.search_tracks(query, limit=5)
                    else:
                        fallback_results = itunes_client.search_tracks(query, limit=5)
                    if fallback_results:
                        match, confidence, _ = _discovery_score_candidates(
                            cleaned_title, cleaned_artist, source_duration, fallback_results
                        )
                        if match and confidence >= min_confidence:
                            matched_track = match
                            best_confidence = confidence
                            print(f"✅ Strategy 3 YouTube match (raw): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

                # Strategy 4: Extended search with higher limit (last resort)
                if not matched_track:
                    print(f"🔄 YouTube Strategy 4: Extended search with limit=50")
                    query = f"{cleaned_artist} {cleaned_title}"
                    if use_spotify:
                        extended_results = spotify_client.search_tracks(query, limit=50)
                    else:
                        extended_results = itunes_client.search_tracks(query, limit=50)
                    if extended_results:
                        match, confidence, _ = _discovery_score_candidates(
                            cleaned_title, cleaned_artist, source_duration, extended_results
                        )
                        if match and confidence >= min_confidence:
                            matched_track = match
                            best_confidence = confidence
                            print(f"✅ Strategy 4 YouTube match (extended): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

                # Create result entry
                result = {
                    'index': i,
                    'yt_track': cleaned_title,
                    'yt_artist': cleaned_artist,
                    'status': '✅ Found' if matched_track else '❌ Not Found',
                    'status_class': 'found' if matched_track else 'not-found',
                    'spotify_track': matched_track.name if matched_track else '',
                    'spotify_artist': matched_track.artists[0] if matched_track else '',
                    'spotify_album': matched_track.album if matched_track else '',
                    'duration': f"{track['duration_ms'] // 60000}:{(track['duration_ms'] % 60000) // 1000:02d}" if track['duration_ms'] else '0:00',
                    'discovery_source': discovery_source,
                    'confidence': best_confidence
                }

                if matched_track:
                    state['spotify_matches'] += 1

                    # Build album data based on provider
                    if use_spotify and best_raw_track:
                        album_data = best_raw_track.get('album', {})
                    else:
                        album_data = {
                            'name': matched_track.album,
                            'album_type': 'album',
                            'images': [{'url': matched_track.image_url}] if hasattr(matched_track, 'image_url') and matched_track.image_url else []
                        }

                    result['matched_data'] = {
                        'id': matched_track.id,
                        'name': matched_track.name,
                        'artists': matched_track.artists,
                        'album': album_data,
                        'duration_ms': matched_track.duration_ms,
                        'source': discovery_source
                    }
                    result['spotify_data'] = result['matched_data']

                    # Save to discovery cache (only high-confidence matches)
                    if best_confidence >= 0.7:
                        try:
                            cache_db = get_database()
                            cache_db.save_discovery_cache_match(
                                cache_key[0], cache_key[1], discovery_source, best_confidence,
                                result['matched_data'], cleaned_title, cleaned_artist
                            )
                            print(f"💾 CACHE SAVED: {cleaned_artist} - {cleaned_title} (confidence: {best_confidence:.3f})")
                        except Exception as cache_err:
                            print(f"⚠️ Cache save error: {cache_err}")

                state['discovery_results'].append(result)

                print(f"  {'✅' if matched_track else '❌'} Track {i+1}/{len(tracks)}: {result['status']}")

            except Exception as e:
                print(f"❌ Error processing track {i}: {e}")
                result = {
                    'index': i,
                    'yt_track': track['name'],
                    'yt_artist': track['artists'][0] if track['artists'] else 'Unknown',
                    'status': '❌ Error',
                    'status_class': 'error',
                    'spotify_track': '',
                    'spotify_artist': '',
                    'spotify_album': '',
                    'duration': '0:00'
                }
                state['discovery_results'].append(result)

        # Complete discovery
        state['phase'] = 'discovered'
        state['status'] = 'complete'
        state['discovery_progress'] = 100

        playlist_name = playlist['name']
        source_label = 'Spotify' if use_spotify else 'iTunes'
        add_activity_item("✅", f"YouTube Discovery Complete ({source_label})", f"'{playlist_name}' - {state['spotify_matches']}/{len(tracks)} tracks found", "Now")

        print(f"✅ YouTube discovery complete ({discovery_source}): {state['spotify_matches']}/{len(tracks)} tracks matched")

    except Exception as e:
        print(f"❌ Error in YouTube discovery worker: {e}")
        state['status'] = 'error'
        state['phase'] = 'fresh'

def _run_listenbrainz_discovery_worker(playlist_mbid):
    """Background worker for ListenBrainz music discovery process (Spotify preferred, iTunes fallback)"""
    try:
        state = listenbrainz_playlist_states[playlist_mbid]
        playlist = state['playlist']
        tracks = playlist['tracks']

        # Determine which provider to use (Spotify preferred, iTunes fallback)
        use_spotify = spotify_client and spotify_client.is_spotify_authenticated()
        discovery_source = 'spotify' if use_spotify else 'itunes'

        # Get iTunes client for fallback
        from core.itunes_client import iTunesClient
        itunes_client = iTunesClient()

        print(f"🔍 Starting {discovery_source} discovery for {len(tracks)} ListenBrainz tracks...")

        # Store the discovery source in state
        state['discovery_source'] = discovery_source

        # Process each track for discovery
        for i, track in enumerate(tracks):
            try:
                # Update progress
                state['discovery_progress'] = int((i / len(tracks)) * 100)

                # Get cleaned track data from ListenBrainz
                cleaned_title = track['track_name']
                cleaned_artist = track['artist_name']
                album_name = track.get('album_name', '')
                duration_ms = track.get('duration_ms', 0)

                print(f"🔍 Searching {discovery_source} for: '{cleaned_artist}' - '{cleaned_title}'")

                # Check discovery cache first
                cache_key = _get_discovery_cache_key(cleaned_title, cleaned_artist)
                try:
                    cache_db = get_database()
                    cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source)
                    if cached_match and _validate_discovery_cache_artist(cleaned_artist, cached_match):
                        print(f"⚡ CACHE HIT [{i+1}/{len(tracks)}]: {cleaned_artist} - {cleaned_title}")
                        result = {
                            'index': i,
                            'lb_track': cleaned_title,
                            'lb_artist': cleaned_artist,
                            'status': '✅ Found',
                            'status_class': 'found',
                            'spotify_track': cached_match.get('name', ''),
                            'spotify_artist': cached_match.get('artists', [''])[0] if cached_match.get('artists') else '',
                            'spotify_album': cached_match.get('album', {}).get('name', '') if isinstance(cached_match.get('album'), dict) else cached_match.get('album', ''),
                            'duration': f"{duration_ms // 60000}:{(duration_ms % 60000) // 1000:02d}" if duration_ms else '0:00',
                            'discovery_source': discovery_source,
                            'matched_data': cached_match,
                            'spotify_data': cached_match
                        }
                        state['spotify_matches'] += 1
                        state['discovery_results'].append(result)
                        continue
                except Exception as cache_err:
                    print(f"⚠️ Cache lookup error: {cache_err}")

                # Try multiple search strategies using matching engine
                matched_track = None
                best_confidence = 0.0
                best_raw_track = None
                min_confidence = 0.9
                source_duration = duration_ms or 0

                # Strategy 1: Use matching_engine search queries
                try:
                    temp_track = type('TempTrack', (), {
                        'name': cleaned_title,
                        'artists': [cleaned_artist],
                        'album': album_name if album_name else None
                    })()
                    search_queries = matching_engine.generate_download_queries(temp_track)
                    print(f"🔍 Generated {len(search_queries)} search queries for ListenBrainz track")
                except Exception as e:
                    print(f"⚠️ Matching engine failed for ListenBrainz, falling back to basic query: {e}")
                    search_queries = [f"{cleaned_artist} {cleaned_title}", cleaned_title]

                for query_idx, search_query in enumerate(search_queries):
                    try:
                        print(f"🔍 ListenBrainz query {query_idx + 1}/{len(search_queries)}: {search_query}")

                        raw_results = None
                        search_results = None

                        if use_spotify:
                            raw_results = spotify_client.sp.search(q=search_query, type='track', limit=10)
                            if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']:
                                continue
                            search_results = spotify_client.search_tracks(search_query, limit=10)
                        else:
                            search_results = itunes_client.search_tracks(search_query, limit=10)

                        if not search_results:
                            continue

                        # Score all results using the matching engine
                        match, confidence, match_idx = _discovery_score_candidates(
                            cleaned_title, cleaned_artist, source_duration, search_results
                        )

                        if match and confidence > best_confidence and confidence >= min_confidence:
                            best_confidence = confidence
                            matched_track = match
                            if use_spotify and raw_results:
                                best_raw_track = raw_results['tracks']['items'][match_idx] if match_idx < len(raw_results['tracks']['items']) else None
                            else:
                                best_raw_track = None
                            print(f"✅ New best ListenBrainz match: {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

                        if best_confidence >= 0.9:
                            print(f"🎯 High confidence ListenBrainz match found ({best_confidence:.3f}), stopping search")
                            break

                    except Exception as e:
                        print(f"❌ Error in ListenBrainz search for query '{search_query}': {e}")
                        continue

                if matched_track:
                    print(f"✅ Strategy 1 ListenBrainz match: {matched_track.artists[0]} - {matched_track.name} (confidence: {best_confidence:.3f})")

                # Strategy 2: Swapped search (if first failed) - score results properly
                if not matched_track:
                    print("🔄 ListenBrainz Strategy 2: Trying swapped search (artist/title reversed)")
                    if use_spotify:
                        query = f"artist:{cleaned_title} track:{cleaned_artist}"
                        fallback_results = spotify_client.search_tracks(query, limit=5)
                    else:
                        query = f"{cleaned_title} {cleaned_artist}"
                        fallback_results = itunes_client.search_tracks(query, limit=5)
                    if fallback_results:
                        match, confidence, _ = _discovery_score_candidates(
                            cleaned_title, cleaned_artist, source_duration, fallback_results
                        )
                        if match and confidence >= min_confidence:
                            matched_track = match
                            best_confidence = confidence
                            print(f"✅ Strategy 2 ListenBrainz match (swapped): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

                # Strategy 3: Album-based search (if still failed and we have album name) - score results properly
                if not matched_track and album_name:
                    print(f"🔄 ListenBrainz Strategy 3: Trying album-based search: '{cleaned_artist} {album_name} {cleaned_title}'")
                    if use_spotify:
                        query = f"artist:{cleaned_artist} album:{album_name} track:{cleaned_title}"
                        fallback_results = spotify_client.search_tracks(query, limit=5)
                    else:
                        query = f"{cleaned_artist} {album_name} {cleaned_title}"
                        fallback_results = itunes_client.search_tracks(query, limit=5)
                    if fallback_results:
                        match, confidence, _ = _discovery_score_candidates(
                            cleaned_title, cleaned_artist, source_duration, fallback_results
                        )
                        if match and confidence >= min_confidence:
                            matched_track = match
                            best_confidence = confidence
                            print(f"✅ Strategy 3 ListenBrainz match (album): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

                # Strategy 4: Extended search with higher limit (last resort)
                if not matched_track:
                    print(f"🔄 ListenBrainz Strategy 4: Extended search with limit=50")
                    query = f"{cleaned_artist} {cleaned_title}"
                    if use_spotify:
                        extended_results = spotify_client.search_tracks(query, limit=50)
                    else:
                        extended_results = itunes_client.search_tracks(query, limit=50)
                    if extended_results:
                        match, confidence, _ = _discovery_score_candidates(
                            cleaned_title, cleaned_artist, source_duration, extended_results
                        )
                        if match and confidence >= min_confidence:
                            matched_track = match
                            best_confidence = confidence
                            print(f"✅ Strategy 4 ListenBrainz match (extended): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

                # Create result entry
                result = {
                    'index': i,
                    'lb_track': cleaned_title,
                    'lb_artist': cleaned_artist,
                    'status': '✅ Found' if matched_track else '❌ Not Found',
                    'status_class': 'found' if matched_track else 'not-found',
                    'spotify_track': matched_track.name if matched_track else '',
                    'spotify_artist': matched_track.artists[0] if matched_track else '',
                    'spotify_album': matched_track.album if matched_track else '',
                    'duration': f"{duration_ms // 60000}:{(duration_ms % 60000) // 1000:02d}" if duration_ms else '0:00',
                    'discovery_source': discovery_source,
                    'confidence': best_confidence
                }

                if matched_track:
                    state['spotify_matches'] += 1

                    # Build album data based on provider
                    if use_spotify and best_raw_track:
                        album_data = best_raw_track.get('album', {})
                    else:
                        album_data = {
                            'name': matched_track.album,
                            'album_type': 'album',
                            'images': [{'url': matched_track.image_url}] if hasattr(matched_track, 'image_url') and matched_track.image_url else []
                        }

                    result['matched_data'] = {
                        'id': matched_track.id,
                        'name': matched_track.name,
                        'artists': matched_track.artists,
                        'album': album_data,
                        'duration_ms': matched_track.duration_ms,
                        'source': discovery_source
                    }
                    result['spotify_data'] = result['matched_data']

                    # Save to discovery cache (only high-confidence matches)
                    if best_confidence >= 0.7:
                        try:
                            cache_db = get_database()
                            cache_db.save_discovery_cache_match(
                                cache_key[0], cache_key[1], discovery_source, best_confidence,
                                result['matched_data'], cleaned_title, cleaned_artist
                            )
                            print(f"💾 CACHE SAVED: {cleaned_artist} - {cleaned_title} (confidence: {best_confidence:.3f})")
                        except Exception as cache_err:
                            print(f"⚠️ Cache save error: {cache_err}")

                state['discovery_results'].append(result)

                print(f"  {'✅' if matched_track else '❌'} Track {i+1}/{len(tracks)}: {result['status']}")

            except Exception as e:
                print(f"❌ Error processing track {i}: {e}")
                result = {
                    'index': i,
                    'lb_track': track['track_name'],
                    'lb_artist': track['artist_name'],
                    'status': '❌ Error',
                    'status_class': 'error',
                    'spotify_track': '',
                    'spotify_artist': '',
                    'spotify_album': '',
                    'duration': '0:00'
                }
                state['discovery_results'].append(result)

        # Complete discovery
        state['phase'] = 'discovered'
        state['status'] = 'complete'
        state['discovery_progress'] = 100

        playlist_name = playlist['name']
        source_label = 'Spotify' if use_spotify else 'iTunes'
        add_activity_item("✅", f"ListenBrainz Discovery Complete ({source_label})", f"'{playlist_name}' - {state['spotify_matches']}/{len(tracks)} tracks found", "Now")

        print(f"✅ ListenBrainz discovery complete ({discovery_source}): {state['spotify_matches']}/{len(tracks)} tracks matched")

    except Exception as e:
        print(f"❌ Error in ListenBrainz discovery worker: {e}")
        state['status'] = 'error'
        state['phase'] = 'fresh'

def _calculate_similarity(str1, str2):
    """Calculate string similarity using simple character overlap"""
    if not str1 or not str2:
        return 0
    
    # Convert to lowercase and remove extra spaces
    str1 = str1.lower().strip()
    str2 = str2.lower().strip()
    
    if str1 == str2:
        return 1.0
    
    # Calculate character overlap
    set1 = set(str1.replace(' ', ''))
    set2 = set(str2.replace(' ', ''))
    
    if not set1 or not set2:
        return 0
    
    intersection = len(set1.intersection(set2))
    union = len(set1.union(set2))
    
    return intersection / union if union > 0 else 0

@app.route('/api/youtube/sync/start/<url_hash>', methods=['POST'])
def start_youtube_sync(url_hash):
    """Start sync process for a YouTube playlist using discovered Spotify tracks"""
    try:
        if url_hash not in youtube_playlist_states:
            return jsonify({"error": "YouTube playlist not found"}), 404
        
        state = youtube_playlist_states[url_hash]
        state['last_accessed'] = time.time()  # Update access time
        
        if state['phase'] not in ['discovered', 'sync_complete']:
            return jsonify({"error": "YouTube playlist not ready for sync"}), 400
        
        # Convert discovery results to Spotify tracks format
        spotify_tracks = convert_youtube_results_to_spotify_tracks(state['discovery_results'])
        
        if not spotify_tracks:
            return jsonify({"error": "No Spotify matches found for sync"}), 400
        
        # Create a temporary playlist ID for sync tracking
        sync_playlist_id = f"youtube_{url_hash}"
        playlist_name = state['playlist']['name']
        
        # Add activity for sync start
        add_activity_item("🔄", "YouTube Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now")
        
        # Update YouTube state
        state['phase'] = 'syncing'
        state['sync_playlist_id'] = sync_playlist_id
        state['sync_progress'] = {}
        
        # Start the sync using existing sync infrastructure
        sync_data = {
            'playlist_id': sync_playlist_id,
            'playlist_name': playlist_name,
            'tracks': spotify_tracks
        }
        
        with sync_lock:
            sync_states[sync_playlist_id] = {"status": "starting", "progress": {}}
        
        # Submit sync task
        future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks)
        active_sync_workers[sync_playlist_id] = future
        
        print(f"🔄 Started YouTube sync for: {playlist_name} ({len(spotify_tracks)} tracks)")
        return jsonify({"success": True, "sync_playlist_id": sync_playlist_id})
        
    except Exception as e:
        print(f"❌ Error starting YouTube sync: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/youtube/sync/status/<url_hash>', methods=['GET'])
def get_youtube_sync_status(url_hash):
    """Get sync status for a YouTube playlist"""
    try:
        if url_hash not in youtube_playlist_states:
            return jsonify({"error": "YouTube playlist not found"}), 404
        
        state = youtube_playlist_states[url_hash]
        state['last_accessed'] = time.time()  # Update access time
        sync_playlist_id = state.get('sync_playlist_id')
        
        if not sync_playlist_id:
            return jsonify({"error": "No sync in progress"}), 404
        
        # Get sync status from existing sync infrastructure
        with sync_lock:
            sync_state = sync_states.get(sync_playlist_id, {})
        
        response = {
            'phase': state['phase'],
            'sync_status': sync_state.get('status', 'unknown'),
            'progress': sync_state.get('progress', {}),
            'complete': sync_state.get('status') == 'finished',
            'error': sync_state.get('error')
        }
        
        # Update YouTube state if sync completed
        if sync_state.get('status') == 'finished':
            state['phase'] = 'sync_complete'
            state['sync_progress'] = sync_state.get('progress', {})
            # Add activity for sync completion
            playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist')
            add_activity_item("🔄", "Sync Complete", f"YouTube playlist '{playlist_name}' synced successfully", "Now")
        elif sync_state.get('status') == 'error':
            state['phase'] = 'discovered'  # Revert on error
            playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist')
            add_activity_item("❌", "Sync Failed", f"YouTube playlist '{playlist_name}' sync failed", "Now")
        
        return jsonify(response)
        
    except Exception as e:
        print(f"❌ Error getting YouTube sync status: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/youtube/sync/cancel/<url_hash>', methods=['POST'])
def cancel_youtube_sync(url_hash):
    """Cancel sync for a YouTube playlist"""
    try:
        if url_hash not in youtube_playlist_states:
            return jsonify({"error": "YouTube playlist not found"}), 404
        
        state = youtube_playlist_states[url_hash]
        state['last_accessed'] = time.time()  # Update access time
        sync_playlist_id = state.get('sync_playlist_id')
        
        if sync_playlist_id:
            # Cancel the sync using existing sync infrastructure
            with sync_lock:
                sync_states[sync_playlist_id] = {"status": "cancelled"}
            
            # Clean up sync worker
            if sync_playlist_id in active_sync_workers:
                del active_sync_workers[sync_playlist_id]
        
        # Revert YouTube state
        state['phase'] = 'discovered'
        state['sync_playlist_id'] = None
        state['sync_progress'] = {}
        
        return jsonify({"success": True, "message": "YouTube sync cancelled"})
        
    except Exception as e:
        print(f"❌ Error cancelling YouTube sync: {e}")
        return jsonify({"error": str(e)}), 500

# New YouTube Playlist Management Endpoints (for persistent state)

@app.route('/api/youtube/playlists', methods=['GET'])
def get_all_youtube_playlists():
    """Get all stored YouTube playlists for frontend hydration (similar to Spotify playlists)"""
    try:
        playlists = []
        current_time = time.time()
        
        for url_hash, state in youtube_playlist_states.items():
            # Skip mirrored playlist entries — they have their own hydration
            if url_hash.startswith('mirrored_'):
                continue
            # Update access time when requested
            state['last_accessed'] = current_time

            # Return essential data for card recreation
            playlist_info = {
                'url_hash': url_hash,
                'url': state['url'],
                'playlist': state['playlist'],
                'phase': state['phase'],
                'status': state['status'],
                'discovery_progress': state['discovery_progress'],
                'spotify_matches': state['spotify_matches'],
                'spotify_total': state['spotify_total'],
                'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'),
                'download_process_id': state.get('download_process_id'),
                'created_at': state['created_at'],
                'last_accessed': state['last_accessed']
            }
            playlists.append(playlist_info)
        
        print(f"📋 Returning {len(playlists)} stored YouTube playlists for hydration")
        return jsonify({"playlists": playlists})
        
    except Exception as e:
        print(f"❌ Error getting YouTube playlists: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/youtube/state/<url_hash>', methods=['GET'])
def get_youtube_playlist_state(url_hash):
    """Get specific YouTube playlist state (detailed version of status endpoint)"""
    try:
        if url_hash not in youtube_playlist_states:
            return jsonify({"error": "YouTube playlist not found"}), 404
        
        state = youtube_playlist_states[url_hash]
        state['last_accessed'] = time.time()
        
        # Return full state information (including results for modal hydration)
        response = {
            'url_hash': url_hash,
            'url': state['url'],
            'playlist': state['playlist'],
            'phase': state['phase'],
            'status': state['status'],
            'discovery_progress': state['discovery_progress'],
            'spotify_matches': state['spotify_matches'],
            'spotify_total': state['spotify_total'],
            'discovery_results': state['discovery_results'],
            'sync_playlist_id': state['sync_playlist_id'],
            'converted_spotify_playlist_id': state['converted_spotify_playlist_id'],
            'sync_progress': state['sync_progress'],
            'created_at': state['created_at'],
            'last_accessed': state['last_accessed']
        }
        
        return jsonify(response)
        
    except Exception as e:
        print(f"❌ Error getting YouTube playlist state: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/youtube/reset/<url_hash>', methods=['POST'])
def reset_youtube_playlist(url_hash):
    """Reset YouTube playlist to fresh phase (clear discovery/sync data)"""
    try:
        if url_hash not in youtube_playlist_states:
            return jsonify({"error": "YouTube playlist not found"}), 404
        
        state = youtube_playlist_states[url_hash]
        
        # Stop any active discovery
        if 'discovery_future' in state and state['discovery_future']:
            state['discovery_future'].cancel()
        
        # Reset state to fresh (preserve original playlist data)
        state['phase'] = 'fresh'
        state['status'] = 'parsed'
        state['discovery_results'] = []
        state['discovery_progress'] = 0
        state['spotify_matches'] = 0
        state['sync_playlist_id'] = None
        state['converted_spotify_playlist_id'] = None
        state['sync_progress'] = {}
        state['discovery_future'] = None
        state['last_accessed'] = time.time()
        
        print(f"🔄 Reset YouTube playlist to fresh phase: {state['playlist']['name']}")
        return jsonify({"success": True, "message": "Playlist reset to fresh state"})
        
    except Exception as e:
        print(f"❌ Error resetting YouTube playlist: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/youtube/delete/<url_hash>', methods=['DELETE'])
def delete_youtube_playlist(url_hash):
    """Remove YouTube playlist from backend storage entirely"""
    try:
        if url_hash not in youtube_playlist_states:
            return jsonify({"error": "YouTube playlist not found"}), 404
        
        state = youtube_playlist_states[url_hash]
        
        # Stop any active discovery
        if 'discovery_future' in state and state['discovery_future']:
            state['discovery_future'].cancel()
        
        # Remove from storage
        playlist_name = state['playlist']['name']
        del youtube_playlist_states[url_hash]
        
        print(f"🗑️ Deleted YouTube playlist from backend: {playlist_name}")
        return jsonify({"success": True, "message": f"Playlist '{playlist_name}' deleted"})
        
    except Exception as e:
        print(f"❌ Error deleting YouTube playlist: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/youtube/update_phase/<url_hash>', methods=['POST'])
def update_youtube_playlist_phase(url_hash):
    """Update YouTube playlist phase (used when modal closes to reset from download_complete to discovered)"""
    try:
        if url_hash not in youtube_playlist_states:
            return jsonify({"error": "YouTube playlist not found"}), 404
        
        data = request.get_json()
        if not data or 'phase' not in data:
            return jsonify({"error": "Phase not provided"}), 400
        
        new_phase = data['phase']
        valid_phases = ['fresh', 'parsed', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete']
        
        if new_phase not in valid_phases:
            return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400
        
        state = youtube_playlist_states[url_hash]
        old_phase = state.get('phase', 'unknown')
        state['phase'] = new_phase
        state['last_accessed'] = time.time()
        
        print(f"🔄 Updated YouTube playlist {url_hash} phase: {old_phase} → {new_phase}")
        return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase})
        
    except Exception as e:
        print(f"❌ Error updating YouTube playlist phase: {e}")
        return jsonify({"error": str(e)}), 500

def convert_youtube_results_to_spotify_tracks(discovery_results):
    """Convert YouTube discovery results to Spotify tracks format for sync"""
    spotify_tracks = []

    for result in discovery_results:
        # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery)
        if result.get('spotify_data'):
            spotify_data = result['spotify_data']

            # Create track object matching the expected format
            track = {
                'id': spotify_data['id'],
                'name': spotify_data['name'],
                'artists': spotify_data['artists'],
                'album': spotify_data['album'],
                'duration_ms': spotify_data.get('duration_ms', 0)
            }
            spotify_tracks.append(track)
        elif result.get('spotify_track') and result.get('status_class') == 'found':
            # Build from individual fields (automatic discovery format)
            track = {
                'id': result.get('spotify_id', 'unknown'),
                'name': result.get('spotify_track', 'Unknown Track'),
                'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'],
                'album': result.get('spotify_album', 'Unknown Album'),
                'duration_ms': 0
            }
            spotify_tracks.append(track)

    print(f"🔄 Converted {len(spotify_tracks)} YouTube matches to Spotify tracks for sync")
    return spotify_tracks


# Add these new endpoints to the end of web_server.py

def _run_sync_task(playlist_id, playlist_name, tracks_json):
    """The actual sync function that runs in the background thread."""
    global sync_states, sync_service
    
    task_start_time = time.time()
    print(f"🚀 [TIMING] _run_sync_task STARTED for playlist '{playlist_name}' at {time.strftime('%H:%M:%S')}")
    print(f"📊 Received {len(tracks_json)} tracks from frontend")

    try:
        # Recreate a Playlist object from the JSON data sent by the frontend
        # This avoids needing to re-fetch it from Spotify
        print(f"🔄 Converting JSON tracks to SpotifyTrack objects...")

        # Store original track data with full album objects (for wishlist with cover art)
        original_tracks_map = {}
        for t in tracks_json:
            track_id = t.get('id', '')
            if track_id:
                original_tracks_map[track_id] = t

        tracks = []
        for i, t in enumerate(tracks_json):
            # Handle album field - extract name if it's a dictionary
            raw_album = t.get('album', '')
            if isinstance(raw_album, dict) and 'name' in raw_album:
                album_name = raw_album['name']
            elif isinstance(raw_album, str):
                album_name = raw_album
            else:
                album_name = str(raw_album)
            
            # Create SpotifyTrack objects with proper default values for missing fields
            track = SpotifyTrack(
                id=t.get('id', ''),  # Provide default empty string
                name=t.get('name', ''),
                artists=t.get('artists', []),
                album=album_name,
                duration_ms=t.get('duration_ms', 0),
                popularity=t.get('popularity', 0),  # Default value
                preview_url=t.get('preview_url'),
                external_urls=t.get('external_urls')
            )
            tracks.append(track)
            if i < 3:  # Log first 3 tracks for debugging
                print(f"  Track {i+1}: '{track.name}' by {track.artists}")
        
        print(f"✅ Created {len(tracks)} SpotifyTrack objects")
        
        playlist = SpotifyPlaylist(
            id=playlist_id, 
            name=playlist_name, 
            description=None,  # Not needed for sync
            owner="web_user",  # Placeholder  
            public=False,      # Default
            collaborative=False,  # Default
            tracks=tracks, 
            total_tracks=len(tracks)
        )
        print(f"✅ Created SpotifyPlaylist object: '{playlist.name}' with {playlist.total_tracks} tracks")

        first_callback_time = [None]  # Use list to allow modification in nested function
        
        def progress_callback(progress):
            """Callback to update the shared state."""
            if first_callback_time[0] is None:
                first_callback_time[0] = time.time()
                first_callback_duration = (first_callback_time[0] - task_start_time) * 1000
                print(f"⏱️ [TIMING] FIRST progress callback at {time.strftime('%H:%M:%S')} (took {first_callback_duration:.1f}ms from start)")
            
            print(f"⚡ PROGRESS CALLBACK: {progress.current_step} - {progress.current_track}")
            print(f"   📊 Progress: {progress.progress}% ({progress.matched_tracks}/{progress.total_tracks} matched, {progress.failed_tracks} failed)")
            
            with sync_lock:
                sync_states[playlist_id] = {
                    "status": "syncing",
                    "progress": progress.__dict__ # Convert dataclass to dict
                }
                print(f"   ✅ Updated sync_states for {playlist_id}")
                
    except Exception as setup_error:
        print(f"❌ SETUP ERROR in _run_sync_task: {setup_error}")
        import traceback
        traceback.print_exc()
        with sync_lock:
            sync_states[playlist_id] = {
                "status": "error",
                "error": f"Setup error: {str(setup_error)}"
            }
        return

    try:
        print(f"🔧 Setting up sync service...")
        print(f"   sync_service available: {sync_service is not None}")
        
        if sync_service is None:
            raise Exception("sync_service is None - not initialized properly")
            
        # Check sync service components
        print(f"   spotify_client: {sync_service.spotify_client is not None}")
        print(f"   plex_client: {sync_service.plex_client is not None}")
        print(f"   jellyfin_client: {sync_service.jellyfin_client is not None}")
        
        # Check media server connection before starting
        from config.settings import config_manager
        active_server = config_manager.get_active_media_server()
        print(f"   Active media server: {active_server}")
        
        media_client, server_type = sync_service._get_active_media_client()
        print(f"   Media client available: {media_client is not None}")
        
        if media_client:
            is_connected = media_client.is_connected()
            print(f"   Media client connected: {is_connected}")
        
        # Check database access
        try:
            from database.music_database import MusicDatabase
            db = MusicDatabase()
            print(f"   Database initialized: {db is not None}")
        except Exception as db_error:
            print(f"   ❌ Database initialization failed: {db_error}")
        
        print(f"🔄 Attaching progress callback...")
        # Attach the progress callback
        sync_service.set_progress_callback(progress_callback, playlist.name)
        print(f"✅ Progress callback attached for playlist: {playlist.name}")

        # CRITICAL FIX: Add database-only fallback for web context
        # If media client is not connected, patch the sync service to use database-only matching
        if media_client is None or not media_client.is_connected():
            print(f"⚠️ Media client not connected - patching sync service for database-only matching")
            
            # Store original method
            original_find_track = sync_service._find_track_in_media_server
            
            # Create database-only replacement method
            async def database_only_find_track(spotify_track):
                print(f"🗃️ Database-only search for: '{spotify_track.name}' by {spotify_track.artists}")
                try:
                    from database.music_database import MusicDatabase
                    from config.settings import config_manager
                    
                    db = MusicDatabase()
                    active_server = config_manager.get_active_media_server()
                    original_title = spotify_track.name
                    
                    # Try each artist (same logic as original)
                    for artist in spotify_track.artists:
                        # Extract artist name from both string and dict formats
                        if isinstance(artist, str):
                            artist_name = artist
                        elif isinstance(artist, dict) and 'name' in artist:
                            artist_name = artist['name']
                        else:
                            artist_name = str(artist)
                        
                        db_track, confidence = db.check_track_exists(
                            original_title, artist_name,
                            confidence_threshold=0.80,
                            server_source=active_server
                        )

                        if db_track and confidence >= 0.80:
                            print(f"✅ Database match: '{db_track.title}' (confidence: {confidence:.2f})")
                            
                            # Create mock track object for playlist creation
                            class DatabaseTrackMock:
                                def __init__(self, db_track):
                                    self.ratingKey = db_track.id
                                    self.title = db_track.title
                                    self.id = db_track.id
                                    # Add any other attributes needed for playlist creation
                            
                            return DatabaseTrackMock(db_track), confidence
                    
                    print(f"❌ No database match found for: '{original_title}'")
                    return None, 0.0
                    
                except Exception as e:
                    print(f"❌ Database search error: {e}")
                    return None, 0.0
            
            # Patch the method
            sync_service._find_track_in_media_server = database_only_find_track
            print(f"✅ Patched sync service to use database-only matching")

        sync_start_time = time.time()
        setup_duration = (sync_start_time - task_start_time) * 1000
        print(f"⏱️ [TIMING] Setup completed at {time.strftime('%H:%M:%S')} (took {setup_duration:.1f}ms)")
        print(f"🚀 Starting actual sync process with run_async()...")

        # Attach original tracks map to sync_service for wishlist with album images
        sync_service._original_tracks_map = original_tracks_map

        # Run the sync (this is a blocking call within this thread)
        result = run_async(sync_service.sync_playlist(playlist, download_missing=False))
        
        sync_duration = (time.time() - sync_start_time) * 1000
        total_duration = (time.time() - task_start_time) * 1000
        print(f"⏱️ [TIMING] Sync completed at {time.strftime('%H:%M:%S')} (sync: {sync_duration:.1f}ms, total: {total_duration:.1f}ms)")
        print(f"✅ Sync process completed! Result type: {type(result)}")
        print(f"   Result details: matched={getattr(result, 'matched_tracks', 'N/A')}, total={getattr(result, 'total_tracks', 'N/A')}")

        # Update final state on completion
        with sync_lock:
            sync_states[playlist_id] = {
                "status": "finished",
                "progress": result.__dict__,  # Store result as progress for status endpoint compatibility
                "result": result.__dict__  # Keep result for backward compatibility
            }
        print(f"🏁 Sync finished for {playlist_id} - state updated")

        # Emit playlist_synced event for automation engine
        try:
            if automation_engine:
                automation_engine.emit('playlist_synced', {
                    'playlist_name': playlist_name,
                    'total_tracks': str(getattr(result, 'total_tracks', 0)),
                    'matched_tracks': str(getattr(result, 'matched_tracks', 0)),
                    'synced_tracks': str(getattr(result, 'synced_tracks', 0)),
                    'failed_tracks': str(getattr(result, 'failed_tracks', 0)),
                })
        except Exception:
            pass

        # Save sync status to storage/sync_status.json (same as GUI)
        # Handle snapshot_id safely - may not exist in all playlist objects
        snapshot_id = getattr(playlist, 'snapshot_id', None)
        _update_and_save_sync_status(playlist_id, playlist_name, playlist.owner, snapshot_id)

    except Exception as e:
        print(f"❌ SYNC FAILED for {playlist_id}: {e}")
        import traceback
        traceback.print_exc()
        with sync_lock:
            sync_states[playlist_id] = {
                "status": "error",
                "error": str(e)
            }
    finally:
        print(f"🧹 Cleaning up progress callback for {playlist.name}")
        # Clean up the callback
        if sync_service:
            sync_service.clear_progress_callback(playlist.name)
            # Clean up original tracks map
            if hasattr(sync_service, '_original_tracks_map'):
                del sync_service._original_tracks_map
        print(f"✅ Cleanup completed for {playlist_id}")


@app.route('/api/sync/start', methods=['POST'])
def start_playlist_sync():
    """Starts a new sync process for a given playlist."""
    request_start_time = time.time()
    print(f"⏱️ [TIMING] Sync request received at {time.strftime('%H:%M:%S')}")
    
    data = request.get_json()
    playlist_id = data.get('playlist_id')
    playlist_name = data.get('playlist_name')
    tracks_json = data.get('tracks') # Pass the full track list

    if not all([playlist_id, playlist_name, tracks_json]):
        return jsonify({"success": False, "error": "Missing playlist_id, name, or tracks."}), 400
    
    # Add activity for sync start
    add_activity_item("🔄", "Spotify Sync Started", f"'{playlist_name}' - {len(tracks_json)} tracks", "Now")

    logger.info(f"🔄 Starting playlist sync for '{playlist_name}' with {len(tracks_json)} tracks")
    logger.debug(f"Request parsed at {time.strftime('%H:%M:%S')} (took {(time.time()-request_start_time)*1000:.1f}ms)")

    with sync_lock:
        if playlist_id in active_sync_workers and not active_sync_workers[playlist_id].done():
            return jsonify({"success": False, "error": "Sync is already in progress for this playlist."}), 409

        # Initial state
        sync_states[playlist_id] = {"status": "starting", "progress": {}}

        # Submit the task to the thread pool
        thread_submit_time = time.time()
        future = sync_executor.submit(_run_sync_task, playlist_id, playlist_name, tracks_json)
        active_sync_workers[playlist_id] = future
        thread_submit_duration = (time.time() - thread_submit_time) * 1000
        print(f"⏱️ [TIMING] Thread submitted at {time.strftime('%H:%M:%S')} (took {thread_submit_duration:.1f}ms)")

    total_request_time = (time.time() - request_start_time) * 1000
    print(f"⏱️ [TIMING] Request completed at {time.strftime('%H:%M:%S')} (total: {total_request_time:.1f}ms)")
    return jsonify({"success": True, "message": "Sync started."})


@app.route('/api/sync/status/<playlist_id>', methods=['GET'])
def get_sync_status(playlist_id):
    """Polls for the status of an ongoing sync."""
    with sync_lock:
        state = sync_states.get(playlist_id)
        if not state:
            return jsonify({"status": "not_found"}), 404

        # If the task is finished but the state hasn't been updated, check the future
        if state['status'] not in ['finished', 'error'] and playlist_id in active_sync_workers:
            if active_sync_workers[playlist_id].done():
                # The task might have finished between polls, trigger final state update
                # This is handled by the _run_sync_task itself
                pass

        return jsonify(state)


@app.route('/api/sync/cancel', methods=['POST'])
def cancel_playlist_sync():
    """Cancels an ongoing sync process."""
    data = request.get_json()
    playlist_id = data.get('playlist_id')

    if not playlist_id:
        return jsonify({"success": False, "error": "Missing playlist_id."}), 400

    with sync_lock:
        future = active_sync_workers.get(playlist_id)
        if not future or future.done():
            return jsonify({"success": False, "error": "Sync not running or already complete."}), 404

        # The GUI's sync_service has a cancel_sync method. We'll replicate that idea.
        # Since we can't easily stop the thread, we'll set a flag.
        # The elegant solution is to have the sync_service check for a cancellation flag.
        # Your `sync_service.py` already has this logic with `self._cancelled`.
        sync_service.cancel_sync()

        # We can't guarantee immediate stop, but we can update the state
        sync_states[playlist_id] = {"status": "cancelled"}

        # It's best practice to let the task finish and clean itself up.
        # We don't use future.cancel() as it may not work if the task is already running.

    return jsonify({"success": True, "message": "Sync cancellation requested."})

@app.route('/api/sync/test-database', methods=['GET'])
def test_database_access():
    """Test endpoint to verify database connectivity for sync operations"""
    try:
        print(f"🧪 Testing database access for sync operations...")
        
        # Test database initialization
        from database.music_database import MusicDatabase
        db = MusicDatabase()
        print(f"   ✅ Database initialized: {db is not None}")
        
        # Test basic database query
        stats = db.get_database_info_for_server()
        print(f"   ✅ Database stats retrieved: {stats}")
        
        # Test track existence check (like sync service does)
        db_track, confidence = db.check_track_exists("test track", "test artist", confidence_threshold=0.7)
        print(f"   ✅ Track existence check works: found={db_track is not None}, confidence={confidence}")
        
        # Test config manager
        from config.settings import config_manager
        active_server = config_manager.get_active_media_server()
        print(f"   ✅ Active media server: {active_server}")
        
        # Test media clients 
        print(f"   Media clients status:")
        print(f"     plex_client: {plex_client is not None}")
        if plex_client:
            print(f"     plex_client.is_connected(): {plex_client.is_connected()}")
        print(f"     jellyfin_client: {jellyfin_client is not None}")
        if jellyfin_client:
            print(f"     jellyfin_client.is_connected(): {jellyfin_client.is_connected()}")
        
        return jsonify({
            "success": True, 
            "message": "Database access test successful",
            "details": {
                "database_initialized": db is not None,
                "database_stats": stats,
                "active_server": active_server,
                "plex_connected": plex_client.is_connected() if plex_client else False,
                "jellyfin_connected": jellyfin_client.is_connected() if jellyfin_client else False,
            }
        })
        
    except Exception as e:
        print(f"   ❌ Database test failed: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({
            "success": False, 
            "error": str(e),
            "message": "Database access test failed"
        }), 500

# --- Discover Download Snapshot System ---

@app.route('/api/discover_downloads/snapshot', methods=['POST'])
def save_discover_download_snapshot():
    """
    Saves a snapshot of current discover download state for persistence across page refreshes.
    """
    try:
        from datetime import datetime

        data = request.json
        if not data or 'downloads' not in data:
            return jsonify({'success': False, 'error': 'No download data provided'}), 400

        downloads = data['downloads']

        db = get_database()
        db.save_bubble_snapshot('discover_downloads', downloads, profile_id=get_current_profile_id())

        download_count = len(downloads)
        print(f"📸 Saved discover download snapshot: {download_count} downloads")

        return jsonify({
            'success': True,
            'message': f'Snapshot saved with {download_count} downloads',
            'timestamp': datetime.now().isoformat()
        })

    except Exception as e:
        print(f"❌ Error saving discover download snapshot: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/discover_downloads/hydrate', methods=['GET'])
def hydrate_discover_downloads():
    """
    Loads discover downloads with live status by cross-referencing snapshots with active processes.
    """
    try:
        from datetime import datetime, timedelta

        db = get_database()
        snapshot = db.get_bubble_snapshot('discover_downloads', profile_id=get_current_profile_id())

        # Load snapshot if it exists
        if not snapshot:
            return jsonify({
                'success': True,
                'downloads': {},
                'message': 'No snapshots found'
            })

        saved_downloads = snapshot['data']
        snapshot_time = snapshot['timestamp']

        # Clean up old snapshots (older than 48 hours)
        try:
            if snapshot_time:
                snapshot_dt = datetime.fromisoformat(snapshot_time.replace('Z', '+00:00'))
                cutoff = datetime.now() - timedelta(hours=48)
                if snapshot_dt < cutoff:
                    print(f"🧹 Cleaning up old discover download snapshot from {snapshot_time}")
                    db.delete_bubble_snapshot('discover_downloads', profile_id=get_current_profile_id())
                    return jsonify({
                        'success': True,
                        'downloads': {},
                        'message': 'Old snapshot cleaned up'
                    })
        except ValueError as e:
            print(f"⚠️ Error checking discover snapshot age: {e}")

        # Get current active download processes for live status
        current_processes = {}
        try:
            with tasks_lock:
                for batch_id, batch_data in download_batches.items():
                    if batch_data.get('phase') not in ['complete', 'error', 'cancelled']:
                        playlist_id = batch_data.get('playlist_id')
                        if playlist_id:
                            current_processes[playlist_id] = {
                                'status': 'in_progress' if batch_data.get('phase') == 'downloading' else 'analyzing',
                                'batch_id': batch_id,
                                'phase': batch_data.get('phase')
                            }
        except Exception as e:
            print(f"⚠️ Error fetching active processes for discover download hydration: {e}")

        # If no active processes exist, the app likely restarted - clean up snapshots
        if not current_processes:
            print(f"🧹 No active processes found - app likely restarted, cleaning up discover download snapshot")
            db.delete_bubble_snapshot('discover_downloads', profile_id=get_current_profile_id())
            return jsonify({
                'success': True,
                'downloads': {},
                'message': 'No active processes - returning empty downloads'
            })

        # Update download statuses with live data
        hydrated_downloads = {}
        for playlist_id, download_data in saved_downloads.items():
            # Determine current live status
            if playlist_id in current_processes:
                process_info = current_processes[playlist_id]
                live_status = 'in_progress'
                print(f"🔄 Found active process for discover download {playlist_id}: {process_info['phase']}")
            else:
                # No active process - likely completed
                live_status = 'completed'
                print(f"✅ No active process for discover download {playlist_id} - marking as completed")

            # Create updated download entry
            hydrated_downloads[playlist_id] = {
                'name': download_data.get('name'),
                'type': download_data.get('type'),
                'status': live_status,
                'virtualPlaylistId': playlist_id,
                'imageUrl': download_data.get('imageUrl'),
                'startTime': download_data.get('startTime', datetime.now().isoformat())
            }

        download_count = len(hydrated_downloads)
        active_count = sum(1 for d in hydrated_downloads.values() if d['status'] == 'in_progress')
        completed_count = sum(1 for d in hydrated_downloads.values() if d['status'] == 'completed')

        print(f"✅ Hydrated {download_count} discover downloads: {active_count} active, {completed_count} completed")

        return jsonify({
            'success': True,
            'downloads': hydrated_downloads,
            'stats': {
                'total_downloads': download_count,
                'active_downloads': active_count,
                'completed_downloads': completed_count
            }
        })

    except Exception as e:
        print(f"❌ Error hydrating discover downloads: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

# --- Artist Bubble Snapshot System ---

@app.route('/api/artist_bubbles/snapshot', methods=['POST'])
def save_artist_bubble_snapshot():
    """
    Saves a snapshot of current artist bubble state for persistence across page refreshes.
    """
    try:
        from datetime import datetime

        data = request.json
        if not data or 'bubbles' not in data:
            return jsonify({'success': False, 'error': 'No bubble data provided'}), 400

        bubbles = data['bubbles']

        db = get_database()
        db.save_bubble_snapshot('artist_bubbles', bubbles, profile_id=get_current_profile_id())

        bubble_count = len(bubbles)
        print(f"📸 Saved artist bubble snapshot: {bubble_count} artists")

        return jsonify({
            'success': True,
            'message': f'Snapshot saved with {bubble_count} artist bubbles',
            'timestamp': datetime.now().isoformat()
        })

    except Exception as e:
        print(f"❌ Error saving artist bubble snapshot: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/artist_bubbles/hydrate', methods=['GET'])
def hydrate_artist_bubbles():
    """
    Loads artist bubbles with live status by cross-referencing snapshots with active processes.
    """
    try:
        from datetime import datetime, timedelta

        db = get_database()
        snapshot = db.get_bubble_snapshot('artist_bubbles', profile_id=get_current_profile_id())

        # Load snapshot if it exists
        if not snapshot:
            return jsonify({
                'success': True,
                'bubbles': {},
                'message': 'No snapshots found'
            })

        saved_bubbles = snapshot['data']
        snapshot_time = snapshot['timestamp']

        # Clean up old snapshots (older than 48 hours)
        try:
            if snapshot_time:
                snapshot_dt = datetime.fromisoformat(snapshot_time.replace('Z', '+00:00'))
                cutoff = datetime.now() - timedelta(hours=48)
                if snapshot_dt < cutoff:
                    print(f"🧹 Cleaning up old snapshot from {snapshot_time}")
                    db.delete_bubble_snapshot('artist_bubbles', profile_id=get_current_profile_id())
                    return jsonify({
                        'success': True,
                        'bubbles': {},
                        'message': 'Old snapshot cleaned up'
                    })
        except ValueError as e:
            print(f"⚠️ Error checking snapshot age: {e}")

        # Get current active download processes for live status
        current_processes = {}
        try:
            with tasks_lock:
                for batch_id, batch_data in download_batches.items():
                    if batch_data.get('phase') not in ['complete', 'error', 'cancelled']:
                        playlist_id = batch_data.get('playlist_id')
                        if playlist_id:
                            current_processes[playlist_id] = {
                                'status': 'in_progress' if batch_data.get('phase') == 'downloading' else 'analyzing',
                                'batch_id': batch_id,
                                'phase': batch_data.get('phase')
                            }
        except Exception as e:
            print(f"⚠️ Error fetching active processes for hydration: {e}")

        # If no active processes exist, the app likely restarted - clean up snapshots
        if not current_processes:
            print(f"🧹 No active processes found - app likely restarted, cleaning up snapshot")
            db.delete_bubble_snapshot('artist_bubbles', profile_id=get_current_profile_id())
            return jsonify({
                'success': True,
                'bubbles': {},
                'message': 'No active processes - returning empty bubbles'
            })

        # Update bubble statuses with live data
        hydrated_bubbles = {}
        for artist_id, bubble_data in saved_bubbles.items():
            hydrated_bubble = {
                'artist': bubble_data['artist'],
                'downloads': [],
                'hasCompletedDownloads': False
            }
            
            for download in bubble_data.get('downloads', []):
                virtual_playlist_id = download['virtualPlaylistId']
                
                # Determine current live status
                if virtual_playlist_id in current_processes:
                    process_info = current_processes[virtual_playlist_id]
                    live_status = 'in_progress'
                    print(f"🔄 Found active process for {download['album']['name']}: {process_info['phase']}")
                else:
                    # No active process - likely completed
                    live_status = 'view_results'
                    print(f"✅ No active process for {download['album']['name']} - marking as completed")
                
                # Create updated download entry
                updated_download = {
                    'virtualPlaylistId': virtual_playlist_id,
                    'album': download['album'],
                    'albumType': download.get('albumType', 'album'),
                    'status': live_status,
                    'startTime': download.get('startTime', datetime.now().isoformat())
                }
                
                hydrated_bubble['downloads'].append(updated_download)
                
                # Update hasCompletedDownloads flag
                if live_status == 'view_results':
                    hydrated_bubble['hasCompletedDownloads'] = True
            
            # Only include artists that still have downloads
            if hydrated_bubble['downloads']:
                hydrated_bubbles[artist_id] = hydrated_bubble
        
        bubble_count = len(hydrated_bubbles)
        active_count = sum(1 for bubble in hydrated_bubbles.values() 
                          for download in bubble['downloads'] 
                          if download['status'] == 'in_progress')
        completed_count = sum(1 for bubble in hydrated_bubbles.values() 
                             for download in bubble['downloads'] 
                             if download['status'] == 'view_results')
        
        print(f"🔄 Hydrated {bubble_count} artist bubbles: {active_count} active, {completed_count} completed")
        
        return jsonify({
            'success': True,
            'bubbles': hydrated_bubbles,
            'stats': {
                'total_artists': bubble_count,
                'active_downloads': active_count,
                'completed_downloads': completed_count,
                'snapshot_time': snapshot_time
            }
        })
        
    except Exception as e:
        print(f"❌ Error hydrating artist bubbles: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

# --- Search Bubble Snapshot System ---

@app.route('/api/search_bubbles/snapshot', methods=['POST'])
def save_search_bubble_snapshot():
    """
    Saves a snapshot of current search bubble state for persistence across page refreshes.
    """
    try:
        from datetime import datetime

        data = request.json
        if not data or 'bubbles' not in data:
            return jsonify({'success': False, 'error': 'No bubble data provided'}), 400

        bubbles = data['bubbles']

        db = get_database()
        db.save_bubble_snapshot('search_bubbles', bubbles, profile_id=get_current_profile_id())

        bubble_count = len(bubbles)
        print(f"📸 Saved search bubble snapshot: {bubble_count} albums/tracks")

        return jsonify({
            'success': True,
            'message': f'Snapshot saved with {bubble_count} search bubbles',
            'timestamp': datetime.now().isoformat()
        })

    except Exception as e:
        print(f"❌ Error saving search bubble snapshot: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/search_bubbles/hydrate', methods=['GET'])
def hydrate_search_bubbles():
    """
    Loads search bubbles with live status by cross-referencing snapshots with active processes.
    """
    try:
        from datetime import datetime, timedelta

        db = get_database()
        snapshot = db.get_bubble_snapshot('search_bubbles', profile_id=get_current_profile_id())

        # Load snapshot if it exists
        if not snapshot:
            return jsonify({
                'success': True,
                'bubbles': {},
                'message': 'No snapshots found'
            })

        saved_bubbles = snapshot['data']
        snapshot_time = snapshot['timestamp']

        # Clean up old snapshots (older than 48 hours)
        try:
            if snapshot_time:
                snapshot_dt = datetime.fromisoformat(snapshot_time.replace('Z', '+00:00'))
                cutoff = datetime.now() - timedelta(hours=48)
                if snapshot_dt < cutoff:
                    print(f"🧹 Cleaning up old search snapshot from {snapshot_time}")
                    db.delete_bubble_snapshot('search_bubbles', profile_id=get_current_profile_id())
                    return jsonify({
                        'success': True,
                        'bubbles': {},
                        'message': 'Old snapshot cleaned up'
                    })
        except ValueError as e:
            print(f"⚠️ Error checking snapshot age: {e}")

        # Get current active download processes for live status
        current_processes = {}
        try:
            with tasks_lock:
                for batch_id, batch_data in download_batches.items():
                    if batch_data.get('phase') not in ['complete', 'error', 'cancelled']:
                        playlist_id = batch_data.get('playlist_id')
                        if playlist_id:
                            current_processes[playlist_id] = {
                                'status': 'in_progress' if batch_data.get('phase') == 'downloading' else 'analyzing',
                                'batch_id': batch_id,
                                'phase': batch_data.get('phase')
                            }
        except Exception as e:
            print(f"⚠️ Error fetching active processes for hydration: {e}")

        # If no active processes exist, the app likely restarted - clean up snapshots
        if not current_processes:
            print(f"🧹 No active processes found - app likely restarted, cleaning up search snapshot")
            db.delete_bubble_snapshot('search_bubbles', profile_id=get_current_profile_id())
            return jsonify({
                'success': True,
                'bubbles': {},
                'message': 'No active processes - returning empty bubbles'
            })

        # Update bubble statuses with live data (artist-grouped structure)
        hydrated_bubbles = {}
        for artist_name, bubble_data in saved_bubbles.items():
            hydrated_bubble = {
                'artist': bubble_data['artist'],
                'downloads': []
            }

            for download in bubble_data.get('downloads', []):
                virtual_playlist_id = download['virtualPlaylistId']

                # Determine current live status
                if virtual_playlist_id in current_processes:
                    process_info = current_processes[virtual_playlist_id]
                    live_status = 'in_progress'
                    print(f"🔄 Found active process for {download['item']['name']}: {process_info['phase']}")
                else:
                    # No active process - likely completed
                    live_status = 'view_results'
                    print(f"✅ No active process for {download['item']['name']} - marking as completed")

                # Create updated download entry
                updated_download = {
                    'virtualPlaylistId': virtual_playlist_id,
                    'item': download['item'],
                    'type': download.get('type', 'album'),
                    'status': live_status,
                    'startTime': download.get('startTime', datetime.now().isoformat())
                }

                hydrated_bubble['downloads'].append(updated_download)

            # Only include artists that still have downloads
            if hydrated_bubble['downloads']:
                hydrated_bubbles[artist_name] = hydrated_bubble

        bubble_count = len(hydrated_bubbles)
        active_count = sum(1 for bubble in hydrated_bubbles.values()
                          for download in bubble['downloads']
                          if download['status'] == 'in_progress')
        completed_count = sum(1 for bubble in hydrated_bubbles.values()
                             for download in bubble['downloads']
                             if download['status'] == 'view_results')

        print(f"🔄 Hydrated {bubble_count} search bubbles (artists): {active_count} active, {completed_count} completed")

        return jsonify({
            'success': True,
            'bubbles': hydrated_bubbles,
            'stats': {
                'total_items': bubble_count,
                'active_downloads': active_count,
                'completed_downloads': completed_count,
                'snapshot_time': snapshot_time
            }
        })

    except Exception as e:
        print(f"❌ Error hydrating search bubbles: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

# --- Profile API Endpoints ---

@app.route('/api/profiles', methods=['GET'])
def list_profiles():
    """List all profiles"""
    try:
        database = get_database()
        profiles = database.get_all_profiles()
        return jsonify({'success': True, 'profiles': profiles})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)}), 500

@app.route('/api/profiles', methods=['POST'])
def create_profile():
    """Create a new profile (admin only)"""
    try:
        # Check that requester is admin
        database = get_database()
        current = database.get_profile(get_current_profile_id())
        if current and not current['is_admin']:
            return jsonify({'success': False, 'error': 'Admin only'}), 403

        data = request.json or {}
        name = data.get('name', '').strip()
        if not name:
            return jsonify({'success': False, 'error': 'Name is required'}), 400

        avatar_color = data.get('avatar_color', '#6366f1')
        avatar_url = data.get('avatar_url') or None
        pin = data.get('pin')
        pin_hash = None
        if pin:
            from werkzeug.security import generate_password_hash
            pin_hash = generate_password_hash(pin, method='pbkdf2:sha256')

        profile_id = database.create_profile(name, avatar_color, pin_hash, is_admin=False, avatar_url=avatar_url)
        if profile_id is None:
            return jsonify({'success': False, 'error': 'Profile name already exists'}), 409

        return jsonify({'success': True, 'profile_id': profile_id})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)}), 500

@app.route('/api/profiles/<int:profile_id>', methods=['PUT'])
def update_profile(profile_id):
    """Update a profile (admin or self)"""
    try:
        database = get_database()
        current_pid = get_current_profile_id()
        current = database.get_profile(current_pid)
        if not current:
            return jsonify({'success': False, 'error': 'Current profile not found'}), 404

        # Only admin or self can update
        if not current['is_admin'] and current_pid != profile_id:
            return jsonify({'success': False, 'error': 'Unauthorized'}), 403

        data = request.json or {}
        kwargs = {}
        if 'name' in data:
            name = data['name'].strip()
            if not name:
                return jsonify({'success': False, 'error': 'Name cannot be empty'}), 400
            kwargs['name'] = name
        if 'avatar_color' in data:
            kwargs['avatar_color'] = data['avatar_color']
        if 'avatar_url' in data:
            kwargs['avatar_url'] = data['avatar_url'] or None
        if 'is_admin' in data and current['is_admin']:
            # Prevent demoting the last admin
            if not data['is_admin']:
                all_profiles = database.get_all_profiles()
                admin_count = sum(1 for p in all_profiles if p['is_admin'])
                target = database.get_profile(profile_id)
                if target and target['is_admin'] and admin_count <= 1:
                    return jsonify({'success': False, 'error': 'Cannot remove the last admin'}), 400
            kwargs['is_admin'] = int(data['is_admin'])

        success = database.update_profile(profile_id, **kwargs)
        return jsonify({'success': success})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)}), 500

@app.route('/api/profiles/<int:profile_id>', methods=['DELETE'])
def delete_profile(profile_id):
    """Delete a profile (admin only, can't delete self)"""
    try:
        database = get_database()
        current_pid = get_current_profile_id()
        current = database.get_profile(current_pid)

        if not current or not current['is_admin']:
            return jsonify({'success': False, 'error': 'Admin only'}), 403
        if current_pid == profile_id:
            return jsonify({'success': False, 'error': 'Cannot delete your own profile'}), 400

        target = database.get_profile(profile_id)
        if not target:
            return jsonify({'success': False, 'error': 'Profile not found'}), 404

        success = database.delete_profile(profile_id)
        return jsonify({'success': success})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)}), 500

@app.route('/api/profiles/select', methods=['POST'])
def select_profile():
    """Select a profile (validates PIN if set)"""
    try:
        data = request.json or {}
        try:
            profile_id = int(data.get('profile_id', 0))
        except (TypeError, ValueError):
            return jsonify({'success': False, 'error': 'Invalid profile_id'}), 400
        pin = data.get('pin', '')

        if not profile_id:
            return jsonify({'success': False, 'error': 'profile_id required'}), 400

        database = get_database()
        profile = database.get_profile(profile_id)
        if not profile:
            return jsonify({'success': False, 'error': 'Profile not found'}), 404

        # Only enforce PIN when multiple profiles exist (PIN protects against profile switching)
        all_profiles = database.get_all_profiles()
        if len(all_profiles) > 1 and profile['has_pin']:
            if not pin:
                return jsonify({'success': False, 'error': 'PIN required', 'pin_required': True}), 401
            if not database.verify_profile_pin(profile_id, pin):
                return jsonify({'success': False, 'error': 'Invalid PIN'}), 401

        session['profile_id'] = profile_id
        return jsonify({'success': True, 'profile': profile})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)}), 500

@app.route('/api/profiles/current', methods=['GET'])
def get_current_profile():
    """Get the currently selected profile from session"""
    try:
        pid = session.get('profile_id')
        if not pid:
            return jsonify({'success': False, 'error': 'No profile selected'}), 200

        database = get_database()
        profile = database.get_profile(pid)
        if not profile:
            session.pop('profile_id', None)
            return jsonify({'success': False, 'error': 'Profile not found'}), 200

        return jsonify({'success': True, 'profile': profile})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)}), 500

@app.route('/api/profiles/logout', methods=['POST'])
def logout_profile():
    """Clear session — back to profile picker"""
    session.pop('profile_id', None)
    return jsonify({'success': True})

@app.route('/api/profiles/<int:profile_id>/set-pin', methods=['POST'])
def set_profile_pin(profile_id):
    """Set or change PIN for a profile (admin or self)"""
    try:
        database = get_database()
        current_pid = get_current_profile_id()
        current = database.get_profile(current_pid)

        if not current or (not current['is_admin'] and current_pid != profile_id):
            return jsonify({'success': False, 'error': 'Unauthorized'}), 403

        data = request.json or {}
        pin = data.get('pin', '')

        if pin:
            from werkzeug.security import generate_password_hash
            pin_hash = generate_password_hash(pin, method='pbkdf2:sha256')
        else:
            pin_hash = None  # Remove PIN

        success = database.update_profile(profile_id, pin_hash=pin_hash)
        return jsonify({'success': success})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)}), 500

# --- Watchlist API Endpoints ---

@app.route('/api/watchlist/count', methods=['GET'])
def get_watchlist_count():
    """Get the number of artists in the watchlist"""
    try:
        database = get_database()
        count = database.get_watchlist_count(profile_id=get_current_profile_id())

        # Calculate time until next auto-scanning
        next_run_in_seconds = automation_engine.get_system_automation_next_run_seconds('scan_watchlist') if automation_engine else 0

        return jsonify({
            "success": True,
            "count": count,
            "next_run_in_seconds": next_run_in_seconds
        })
    except Exception as e:
        print(f"Error getting watchlist count: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/watchlist/artists', methods=['GET'])
def get_watchlist_artists():
    """Get all artists in the watchlist with cached images"""
    try:
        database = get_database()
        watchlist_artists = database.get_watchlist_artists(profile_id=get_current_profile_id())

        # Convert to JSON serializable format (images are cached from watchlist scans)
        artists_data = []
        for artist in watchlist_artists:
            artists_data.append({
                "id": artist.id,
                "spotify_artist_id": artist.spotify_artist_id,
                "artist_name": artist.artist_name,
                "date_added": artist.date_added.isoformat() if artist.date_added else None,
                "last_scan_timestamp": artist.last_scan_timestamp.isoformat() if artist.last_scan_timestamp else None,
                "created_at": artist.created_at.isoformat() if artist.created_at else None,
                "updated_at": artist.updated_at.isoformat() if artist.updated_at else None,
                "image_url": artist.image_url,  # Cached during watchlist scans
                "itunes_artist_id": artist.itunes_artist_id,  # For iTunes-only artists
                "include_albums": artist.include_albums,
                "include_eps": artist.include_eps,
                "include_singles": artist.include_singles,
                "include_live": artist.include_live,
                "include_remixes": artist.include_remixes,
                "include_acoustic": artist.include_acoustic,
                "include_compilations": artist.include_compilations,
            })

        return jsonify({"success": True, "artists": artists_data})
    except Exception as e:
        print(f"Error getting watchlist artists: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/watchlist/add', methods=['POST'])
def add_to_watchlist():
    """Add an artist to the watchlist"""
    try:
        data = request.get_json()
        artist_id = data.get('artist_id')
        artist_name = data.get('artist_name')

        if not artist_id or not artist_name:
            return jsonify({"success": False, "error": "Missing artist_id or artist_name"}), 400

        database = get_database()
        success = database.add_artist_to_watchlist(artist_id, artist_name, profile_id=get_current_profile_id())

        if success:
            # Detect ID type: iTunes IDs are purely numeric, Spotify IDs are alphanumeric
            is_itunes_id = artist_id.isdigit()
            
            # Fetch and cache artist image immediately
            try:
                if is_itunes_id:
                    # For iTunes artists, fetch image from iTunes API
                    # We look up 'album' entity because 'artist' entity doesn't always have artwork
                    # We fallback to the first album's artwork as the artist image
                    try:
                        itunes_url = f"https://itunes.apple.com/lookup?id={artist_id}&entity=album&limit=5"
                        print(f"🔍 Fetching iTunes artist image: {itunes_url}")
                        resp = requests.get(itunes_url, timeout=5)
                        
                        image_url = None
                        if resp.status_code == 200:
                            data = resp.json()
                            results = data.get('results', [])
                            
                            # Iterate results to find one with artwork
                            for res in results:
                                if 'artworkUrl100' in res:
                                    # Get highest res by replacing dimensions
                                    # iTunes artwork URLs usually end in .../100x100bb.jpg
                                    image_url = res['artworkUrl100'].replace('100x100', '600x600')
                                    break
                        
                        if image_url:
                            database.update_watchlist_artist_image(artist_id, image_url)
                            print(f"✅ Cached iTunes artist image for {artist_name}")
                        else:
                            print(f"⚠️ No artwork found for iTunes artist {artist_name}")
                    except Exception as itunes_error:
                        print(f"⚠️ Error fetching iTunes artwork: {itunes_error}")
                elif spotify_client and spotify_client.is_authenticated():
                    # For Spotify artists, fetch from Spotify API
                    artist_data = spotify_client.get_artist(artist_id)
                    if artist_data and 'images' in artist_data and artist_data['images']:
                        # Get medium-sized image (usually the second one, or first if only one)
                        image_url = None
                        if len(artist_data['images']) > 1:
                            image_url = artist_data['images'][1]['url']
                        else:
                            image_url = artist_data['images'][0]['url']

                        # Update in database
                        if image_url:
                            database.update_watchlist_artist_image(artist_id, image_url)
                            print(f"✅ Cached artist image for {artist_name}")
                        else:
                            print(f"⚠️ No image URL found for {artist_name}")
                    else:
                        print(f"⚠️ No images in Spotify data for {artist_name}")
                else:
                    print(f"⚠️ Spotify client not available for fetching artist image")
            except Exception as img_error:
                # Don't fail the add operation if image fetch fails
                print(f"⚠️ Could not fetch artist image for {artist_name}: {img_error}")

            # Push updated count to this profile's WebSocket room immediately
            try:
                pid = get_current_profile_id()
                socketio.emit('watchlist:count', _build_watchlist_count_payload(profile_id=pid), room=f'profile:{pid}')
            except Exception:
                pass
            try:
                if automation_engine:
                    automation_engine.emit('watchlist_artist_added', {
                        'artist': artist_name,
                        'artist_id': str(artist_id),
                    })
            except Exception:
                pass
            return jsonify({"success": True, "message": f"Added {artist_name} to watchlist"})
        else:
            return jsonify({"success": False, "error": "Failed to add artist to watchlist"}), 500

    except Exception as e:
        print(f"Error adding to watchlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/watchlist/remove', methods=['POST'])
def remove_from_watchlist():
    """Remove an artist from the watchlist"""
    try:
        data = request.get_json()
        artist_id = data.get('artist_id')

        if not artist_id:
            return jsonify({"success": False, "error": "Missing artist_id"}), 400

        database = get_database()
        success = database.remove_artist_from_watchlist(artist_id, profile_id=get_current_profile_id())

        if success:
            # Push updated count to this profile's WebSocket room immediately
            try:
                pid = get_current_profile_id()
                socketio.emit('watchlist:count', _build_watchlist_count_payload(profile_id=pid), room=f'profile:{pid}')
            except Exception:
                pass
            try:
                if automation_engine:
                    automation_engine.emit('watchlist_artist_removed', {
                        'artist': data.get('artist_name', str(artist_id)),
                        'artist_id': str(artist_id),
                    })
            except Exception:
                pass
            return jsonify({"success": True, "message": "Removed artist from watchlist"})
        else:
            return jsonify({"success": False, "error": "Failed to remove artist from watchlist"}), 500

    except Exception as e:
        print(f"Error removing from watchlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/watchlist/add-batch', methods=['POST'])
def add_batch_to_watchlist():
    """Add multiple artists to the watchlist at once"""
    try:
        data = request.get_json()
        artists = data.get('artists', [])

        if not artists or not isinstance(artists, list):
            return jsonify({"success": False, "error": "Missing or invalid artists list"}), 400

        database = get_database()
        added = 0
        skipped = 0

        for artist in artists:
            artist_id = artist.get('artist_id')
            artist_name = artist.get('artist_name')
            if not artist_id or not artist_name:
                continue

            # Check if already watched
            if database.is_artist_in_watchlist(artist_id, profile_id=get_current_profile_id()):
                skipped += 1
                continue

            success = database.add_artist_to_watchlist(artist_id, artist_name, profile_id=get_current_profile_id())
            if success:
                added += 1
                # Cache artist image
                try:
                    is_itunes_id = artist_id.isdigit()
                    if is_itunes_id:
                        itunes_url = f"https://itunes.apple.com/lookup?id={artist_id}&entity=album&limit=5"
                        resp = requests.get(itunes_url, timeout=5)
                        if resp.status_code == 200:
                            results = resp.json().get('results', [])
                            for res in results:
                                if 'artworkUrl100' in res:
                                    image_url = res['artworkUrl100'].replace('100x100', '600x600')
                                    database.update_watchlist_artist_image(artist_id, image_url)
                                    break
                    elif spotify_client and spotify_client.is_authenticated():
                        artist_data = spotify_client.get_artist(artist_id)
                        if artist_data and 'images' in artist_data and artist_data['images']:
                            image_url = artist_data['images'][1]['url'] if len(artist_data['images']) > 1 else artist_data['images'][0]['url']
                            if image_url:
                                database.update_watchlist_artist_image(artist_id, image_url)
                except Exception as img_error:
                    print(f"⚠️ Could not fetch artist image for {artist_name}: {img_error}")

        return jsonify({
            "success": True,
            "added": added,
            "skipped": skipped,
            "message": f"Added {added} artist{'s' if added != 1 else ''} to watchlist ({skipped} already watched)"
        })

    except Exception as e:
        print(f"Error batch adding to watchlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/library/watchlist-all-unwatched', methods=['POST'])
def watchlist_all_unwatched_library_artists():
    """Add all unwatched library artists (that have valid external IDs) to the watchlist"""
    try:
        database = get_database()
        active_source = _get_active_discovery_source()

        # Get ALL unwatched artists from library (no pagination)
        result = database.get_library_artists(
            search_query='',
            letter='all',
            page=1,
            limit=99999,
            watchlist_filter='unwatched',
            profile_id=get_current_profile_id()
        )

        unwatched_artists = result.get('artists', [])
        added = 0
        skipped_no_id = 0
        skipped_already = 0

        for artist in unwatched_artists:
            # Determine the external ID to use based on active source
            artist_id = None
            if active_source == 'spotify' and artist.get('spotify_artist_id'):
                artist_id = artist['spotify_artist_id']
            elif artist.get('itunes_artist_id'):
                artist_id = artist['itunes_artist_id']
            elif artist.get('spotify_artist_id'):
                # Fallback: use spotify if itunes not available
                artist_id = artist['spotify_artist_id']

            if not artist_id:
                skipped_no_id += 1
                continue

            artist_name = artist.get('name', '')
            if not artist_name:
                continue

            # Check if already watched (shouldn't be since we filtered, but safety check)
            if database.is_artist_in_watchlist(artist_id, profile_id=get_current_profile_id()):
                skipped_already += 1
                continue

            success = database.add_artist_to_watchlist(artist_id, artist_name, profile_id=get_current_profile_id())
            if success:
                added += 1
                # Use library thumb_url if available (no HTTP calls needed)
                if artist.get('image_url'):
                    try:
                        database.update_watchlist_artist_image(artist_id, artist['image_url'])
                    except Exception:
                        pass

        total_unwatched = len(unwatched_artists)
        message_parts = [f"Added {added} artist{'s' if added != 1 else ''} to watchlist"]
        if skipped_no_id > 0:
            message_parts.append(f"{skipped_no_id} skipped (no matching ID yet)")

        return jsonify({
            "success": True,
            "added": added,
            "skipped_no_id": skipped_no_id,
            "skipped_already": skipped_already,
            "total_unwatched": total_unwatched,
            "message": " — ".join(message_parts)
        })

    except Exception as e:
        print(f"Error bulk watchlisting library artists: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/watchlist/remove-batch', methods=['POST'])
def remove_batch_from_watchlist():
    """Remove multiple artists from the watchlist"""
    try:
        data = request.get_json()
        artist_ids = data.get('artist_ids', [])

        if not artist_ids or not isinstance(artist_ids, list):
            return jsonify({"success": False, "error": "Missing or invalid artist_ids"}), 400

        database = get_database()
        removed = 0
        for artist_id in artist_ids:
            if database.remove_artist_from_watchlist(artist_id, profile_id=get_current_profile_id()):
                removed += 1

        return jsonify({
            "success": True,
            "removed": removed,
            "message": f"Removed {removed} artist{'s' if removed != 1 else ''} from watchlist"
        })

    except Exception as e:
        print(f"Error batch removing from watchlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/watchlist/check', methods=['POST'])
def check_watchlist_status():
    """Check if an artist is in the watchlist"""
    try:
        data = request.get_json()
        artist_id = data.get('artist_id')
        
        if not artist_id:
            return jsonify({"success": False, "error": "Missing artist_id"}), 400
        
        database = get_database()
        is_watching = database.is_artist_in_watchlist(artist_id, profile_id=get_current_profile_id())

        return jsonify({"success": True, "is_watching": is_watching})
        
    except Exception as e:
        print(f"Error checking watchlist status: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/watchlist/scan', methods=['POST'])
def start_watchlist_scan():
    """Start a watchlist scan for new releases"""
    try:
        # Check if MetadataService can provide a working client (Spotify OR iTunes)
        from core.metadata_service import MetadataService
        metadata_service = MetadataService()
        
        # Get active provider - will be either spotify or itunes
        active_provider = metadata_service.get_active_provider()
        provider_info = metadata_service.get_provider_info()
        
        # Verify we have at least one working provider
        if not provider_info['spotify_authenticated'] and not provider_info['itunes_available']:
            return jsonify({
                "success": False, 
                "error": "No music provider available. Please authenticate Spotify or ensure iTunes is accessible."
            }), 400
        
        logger.info(f"Starting watchlist scan with {active_provider} provider")

        # Check if wishlist auto-processing is currently running (using smart detection)
        if is_wishlist_actually_processing():
            return jsonify({"success": False, "error": "Wishlist auto-processing is currently running. Please wait for it to complete before starting a watchlist scan."}), 409

        # Check if watchlist is already scanning
        if is_watchlist_actually_scanning():
            return jsonify({"success": False, "error": "Watchlist scan is already in progress."}), 409
        
        # Start the scan in a background thread
        scan_profile_id = get_current_profile_id()
        def run_scan():
            _enrichment_was_running = False
            _itunes_enrichment_was_running = False
            try:
                global watchlist_scan_state, watchlist_auto_scanning, watchlist_auto_scanning_timestamp
                from core.watchlist_scanner import WatchlistScanner
                from database.music_database import get_database

                # Set flag and timestamp for manual scan
                import time
                with watchlist_timer_lock:
                    watchlist_auto_scanning = True
                    watchlist_auto_scanning_timestamp = time.time()
                    print(f"🔒 [Manual Watchlist Scan] Flag set at timestamp {watchlist_auto_scanning_timestamp}")

                # Get list of artists to scan (for the current profile)
                database = get_database()
                watchlist_artists = database.get_watchlist_artists(profile_id=scan_profile_id)

                # Apply global overrides if enabled
                _apply_watchlist_global_overrides(watchlist_artists)

                if not watchlist_artists:
                    watchlist_scan_state['status'] = 'completed'
                    watchlist_scan_state['summary'] = {
                        'total_artists': 0,
                        'successful_scans': 0,
                        'new_tracks_found': 0,
                        'tracks_added_to_wishlist': 0
                    }
                    # Reset flag
                    with watchlist_timer_lock:
                        watchlist_auto_scanning = False
                        watchlist_auto_scanning_timestamp = 0
                    return

                # Initialize scanner with MetadataService for cross-provider support
                scanner = WatchlistScanner(metadata_service=metadata_service)
                
                # PROACTIVE ID BACKFILLING (cross-provider support)
                # Before scanning, ensure all artists have IDs for the current provider
                try:
                    active_provider = metadata_service.get_active_provider()
                    print(f"🔍 Checking for missing {active_provider} IDs in watchlist...")
                    scanner._backfill_missing_ids(watchlist_artists, active_provider)
                except Exception as backfill_error:
                    print(f"⚠️ Error during ID backfilling: {backfill_error}")
                    import traceback
                    traceback.print_exc()
                    # Continue with scan even if backfilling fails
                
                # Initialize detailed progress tracking
                watchlist_scan_state.update({
                    'total_artists': len(watchlist_artists),
                    'current_artist_index': 0,
                    'current_artist_name': '',
                    'current_artist_image_url': '',
                    'current_phase': 'starting',
                    'albums_to_check': 0,
                    'albums_checked': 0,
                    'current_album': '',
                    'current_album_image_url': '',
                    'current_track_name': '',
                    'tracks_found_this_scan': 0,
                    'tracks_added_this_scan': 0,
                    'recent_wishlist_additions': []
                })
                
                scan_results = []

                # Pause enrichment workers during scan to reduce API contention
                if spotify_enrichment_worker and not spotify_enrichment_worker.paused:
                    spotify_enrichment_worker.pause()
                    _enrichment_was_running = True
                    print("⏸️ Paused Spotify enrichment worker during watchlist scan")
                if itunes_enrichment_worker and not itunes_enrichment_worker.paused:
                    itunes_enrichment_worker.pause()
                    _itunes_enrichment_was_running = True
                    print("⏸️ Paused iTunes enrichment worker during watchlist scan")

                # Dynamic delay calculation based on scan scope
                lookback_period = scanner._get_lookback_period_setting()
                is_full_discography = (lookback_period == 'all')
                artist_count = len(watchlist_artists)

                base_artist_delay = 2.0
                base_album_delay = 0.5

                # Scale up for full discography (way more albums per artist)
                if is_full_discography:
                    base_artist_delay *= 2.0
                    base_album_delay *= 2.0

                # Scale up further for large artist counts (sustained API pressure)
                if artist_count > 200:
                    base_artist_delay *= 1.5
                    base_album_delay *= 1.25
                elif artist_count > 100:
                    base_artist_delay *= 1.25

                artist_delay = base_artist_delay
                album_delay = base_album_delay
                print(f"📊 Scan parameters: {artist_count} artists, lookback={lookback_period}, "
                      f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album")

                # Circuit breaker: pause scan on consecutive rate-limit failures
                consecutive_failures = 0
                CIRCUIT_BREAKER_THRESHOLD = 3
                circuit_breaker_pause = 60  # seconds, doubles each trigger, max 600s

                for i, artist in enumerate(watchlist_artists):
                    try:
                        # Fetch artist image using provider-aware method
                        artist_image_url = scanner.get_artist_image_url(artist) or ''

                        # Update progress
                        watchlist_scan_state.update({
                            'current_artist_index': i + 1,
                            'current_artist_name': artist.artist_name,
                            'current_artist_image_url': artist_image_url,
                            'current_phase': 'fetching_discography',
                            'albums_to_check': 0,
                            'albums_checked': 0,
                            'current_album': '',
                            'current_album_image_url': '',
                            'current_track_name': ''
                        })

                        # Get artist discography using provider-aware method
                        albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp)

                        if albums is None:
                            scan_results.append(type('ScanResult', (), {
                                'artist_name': artist.artist_name,
                                'spotify_artist_id': artist.spotify_artist_id,
                                'albums_checked': 0,
                                'new_tracks_found': 0,
                                'tracks_added_to_wishlist': 0,
                                'success': False,
                                'error_message': "Failed to get artist discography"
                            })())
                            continue
                        
                        # Update with album count
                        watchlist_scan_state.update({
                            'current_phase': 'checking_albums',
                            'albums_to_check': len(albums),
                            'albums_checked': 0
                        })
                        
                        # Track progress for this artist
                        artist_new_tracks = 0
                        artist_added_tracks = 0
                        
                        # Scan each album
                        for album_index, album in enumerate(albums):
                            try:
                                # Get album tracks using provider-aware method
                                album_data = scanner.metadata_service.get_album(album.id)
                                if not album_data or 'tracks' not in album_data:
                                    continue

                                tracks = album_data['tracks']['items']

                                # Check release type filter (album/EP/single)
                                if not scanner._should_include_release(len(tracks), artist):
                                    continue

                                # Get album image
                                album_image_url = ''
                                if 'images' in album_data and album_data['images']:
                                    album_image_url = album_data['images'][0]['url']

                                watchlist_scan_state.update({
                                    'albums_checked': album_index + 1,
                                    'current_album': album.name,
                                    'current_album_image_url': album_image_url,
                                    'current_phase': f'checking_album_{album_index + 1}_of_{len(albums)}'
                                })

                                # Check each track
                                for track in tracks:
                                    # Check content type filter (live/remix/acoustic/compilation)
                                    if not scanner._should_include_track(track, album_data, artist):
                                        continue

                                    # Update current track being processed
                                    track_name = track.get('name', 'Unknown Track')
                                    watchlist_scan_state['current_track_name'] = track_name

                                    if scanner.is_track_missing_from_library(track):
                                        artist_new_tracks += 1
                                        watchlist_scan_state['tracks_found_this_scan'] += 1

                                        # Add to wishlist
                                        if scanner.add_track_to_wishlist(track, album_data, artist):
                                            artist_added_tracks += 1
                                            watchlist_scan_state['tracks_added_this_scan'] += 1

                                            # Add to recent wishlist additions feed
                                            track_artists = track.get('artists', [])
                                            track_artist_name = track_artists[0].get('name', 'Unknown Artist') if track_artists else 'Unknown Artist'

                                            watchlist_scan_state['recent_wishlist_additions'].insert(0, {
                                                'track_name': track_name,
                                                'artist_name': track_artist_name,
                                                'album_image_url': album_image_url
                                            })

                                            # Keep only last 10
                                            if len(watchlist_scan_state['recent_wishlist_additions']) > 10:
                                                watchlist_scan_state['recent_wishlist_additions'].pop()

                                # Rate-limited delay between albums
                                import time
                                time.sleep(album_delay)

                            except Exception as e:
                                print(f"Error checking album {album.name}: {e}")
                                continue

                        # Update scan timestamp
                        scanner.update_artist_scan_timestamp(artist)

                        # Store result
                        scan_results.append(type('ScanResult', (), {
                            'artist_name': artist.artist_name,
                            'spotify_artist_id': artist.spotify_artist_id,
                            'albums_checked': len(albums),
                            'new_tracks_found': artist_new_tracks,
                            'tracks_added_to_wishlist': artist_added_tracks,
                            'success': True,
                            'error_message': None
                        })())

                        print(f"✅ Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist")

                        # Fetch similar artists for discovery feature
                        # This is critical for the discover page to work
                        try:
                            watchlist_scan_state['current_phase'] = 'fetching_similar_artists'
                            source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id)

                            # If Spotify is authenticated, also require Spotify IDs to be present
                            spotify_authenticated = spotify_client and spotify_client.is_spotify_authenticated()
                            if database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=scan_profile_id):
                                print(f"  Similar artists for {artist.artist_name} are cached and fresh")
                                # Still backfill missing iTunes IDs
                                scanner._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=scan_profile_id)
                            else:
                                print(f"  Fetching similar artists for {artist.artist_name}...")
                                scanner.update_similar_artists(artist, profile_id=scan_profile_id)
                                print(f"  Similar artists updated for {artist.artist_name}")
                        except Exception as similar_error:
                            print(f"  ⚠️ Failed to update similar artists for {artist.artist_name}: {similar_error}")

                        # Delay between artists
                        if i < len(watchlist_artists) - 1:
                            watchlist_scan_state['current_phase'] = 'rate_limiting'
                            time.sleep(artist_delay)

                        # Reset circuit breaker on successful artist scan
                        consecutive_failures = 0
                        circuit_breaker_pause = 60

                    except Exception as e:
                        print(f"Error scanning artist {artist.artist_name}: {e}")

                        # Circuit breaker: detect consecutive rate-limit failures
                        error_str = str(e).lower()
                        if "429" in error_str or "rate limit" in error_str:
                            consecutive_failures += 1
                            if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD:
                                print(f"🛑 Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s")
                                watchlist_scan_state['current_phase'] = 'circuit_breaker_pause'
                                time.sleep(circuit_breaker_pause)
                                circuit_breaker_pause = min(circuit_breaker_pause * 2, 600)
                                consecutive_failures = 0
                        else:
                            consecutive_failures = 0

                        scan_results.append(type('ScanResult', (), {
                            'artist_name': artist.artist_name,
                            'spotify_artist_id': artist.spotify_artist_id,
                            'albums_checked': 0,
                            'new_tracks_found': 0,
                            'tracks_added_to_wishlist': 0,
                            'success': False,
                            'error_message': str(e)
                        })())

                # Store final results
                watchlist_scan_state['status'] = 'completed'
                watchlist_scan_state['results'] = scan_results
                watchlist_scan_state['completed_at'] = datetime.now()
                watchlist_scan_state['current_phase'] = 'completed'
                
                # Calculate summary
                successful_scans = [r for r in scan_results if r.success]
                total_new_tracks = sum(r.new_tracks_found for r in successful_scans)
                total_added_to_wishlist = sum(r.tracks_added_to_wishlist for r in successful_scans)
                
                watchlist_scan_state['summary'] = {
                    'total_artists': len(scan_results),
                    'successful_scans': len(successful_scans),
                    'new_tracks_found': total_new_tracks,
                    'tracks_added_to_wishlist': total_added_to_wishlist
                }
                
                print(f"Watchlist scan completed: {len(successful_scans)}/{len(scan_results)} artists scanned successfully")
                print(f"Found {total_new_tracks} new tracks, added {total_added_to_wishlist} to wishlist")

                # Populate discovery pool from similar artists
                print("🎵 Starting discovery pool population...")
                watchlist_scan_state['current_phase'] = 'populating_discovery_pool'
                try:
                    scanner.populate_discovery_pool(profile_id=scan_profile_id)
                    print("✅ Discovery pool population complete")
                except Exception as discovery_error:
                    print(f"⚠️ Error populating discovery pool: {discovery_error}")
                    import traceback
                    traceback.print_exc()

                # Update ListenBrainz playlists cache
                print("🧠 Starting ListenBrainz playlists update...")
                watchlist_scan_state['current_phase'] = 'updating_listenbrainz'
                try:
                    from core.listenbrainz_manager import ListenBrainzManager
                    lb_manager = ListenBrainzManager(str(get_database().database_path))
                    lb_result = lb_manager.update_all_playlists()
                    if lb_result.get('success'):
                        summary = lb_result.get('summary', {})
                        print(f"✅ ListenBrainz update complete: {summary}")
                    else:
                        print(f"⚠️ ListenBrainz update skipped: {lb_result.get('error')}")
                except Exception as lb_error:
                    print(f"⚠️ Error updating ListenBrainz: {lb_error}")
                    import traceback
                    traceback.print_exc()

                # Update current seasonal playlist (weekly refresh)
                print("🎃 Starting seasonal content update...")
                watchlist_scan_state['current_phase'] = 'updating_seasonal'
                try:
                    from core.seasonal_discovery import get_seasonal_discovery_service
                    seasonal_service = get_seasonal_discovery_service(spotify_client, database)

                    # Only update the current active season
                    current_season = seasonal_service.get_current_season()
                    if current_season:
                        if seasonal_service.should_populate_seasonal_content(current_season, days_threshold=7):
                            print(f"🎃 Updating {current_season} seasonal content...")
                            seasonal_service.populate_seasonal_content(current_season)
                            seasonal_service.curate_seasonal_playlist(current_season)
                            print(f"✅ {current_season.capitalize()} seasonal content updated")
                        else:
                            print(f"⏭️ {current_season.capitalize()} seasonal content recently updated, skipping")
                    else:
                        print("ℹ️ No active season at this time")
                except Exception as seasonal_error:
                    print(f"⚠️ Error updating seasonal content: {seasonal_error}")
                    import traceback
                    traceback.print_exc()

            except Exception as e:
                print(f"Error during watchlist scan: {e}")
                watchlist_scan_state['status'] = 'error'
                watchlist_scan_state['error'] = str(e)

            finally:
                # Resume enrichment workers if we paused them
                if _enrichment_was_running and spotify_enrichment_worker:
                    spotify_enrichment_worker.resume()
                    print("▶️ Resumed Spotify enrichment worker after watchlist scan")
                if _itunes_enrichment_was_running and itunes_enrichment_worker:
                    itunes_enrichment_worker.resume()
                    print("▶️ Resumed iTunes enrichment worker after watchlist scan")

                # Always reset flag when scan completes (success or error)
                with watchlist_timer_lock:
                    watchlist_auto_scanning = False
                    watchlist_auto_scanning_timestamp = 0
                    print("🔓 [Manual Watchlist Scan] Flag reset - scan complete")
        
        # Initialize scan state
        global watchlist_scan_state
        watchlist_scan_state = {
            'status': 'scanning',
            'started_at': datetime.now(),
            'results': [],
            'summary': {},
            'error': None
        }
        
        # Start scan in background
        thread = threading.Thread(target=run_scan)
        thread.daemon = True
        thread.start()
        
        return jsonify({"success": True, "message": "Watchlist scan started"})
        
    except Exception as e:
        print(f"Error starting watchlist scan: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/watchlist/scan/status', methods=['GET'])
def get_watchlist_scan_status():
    """Get the current status of watchlist scanning"""
    try:
        global watchlist_scan_state
        if 'watchlist_scan_state' not in globals():
            return jsonify({
                "success": True,
                "status": "idle",
                "summary": {}
            })

        # Convert datetime objects to ISO format for JSON serialization
        state = watchlist_scan_state.copy()
        if 'started_at' in state and state['started_at']:
            state['started_at'] = state['started_at'].isoformat()
        if 'completed_at' in state and state['completed_at']:
            state['completed_at'] = state['completed_at'].isoformat()

        # Remove results array - it contains ScanResult objects that aren't JSON serializable
        # The summary already contains the aggregate data we need
        if 'results' in state:
            del state['results']

        return jsonify({"success": True, **state})

    except Exception as e:
        print(f"Error getting watchlist scan status: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

# Similar Artists Update State
similar_artists_update_state = {
    'status': 'idle',  # idle, running, completed, error
    'artists_processed': 0,
    'total_artists': 0,
    'current_artist': None,
    'error': None
}
similar_artists_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="SimilarArtistsUpdate")

@app.route('/api/watchlist/update-similar-artists', methods=['POST'])
def update_similar_artists_endpoint():
    """Update similar artists for all watchlist artists (for discovery feature)"""
    try:
        global similar_artists_update_state

        if similar_artists_update_state['status'] == 'running':
            return jsonify({"success": False, "error": "Similar artists update already in progress"}), 409

        if not spotify_client or not spotify_client.is_authenticated():
            return jsonify({"success": False, "error": "Spotify client not available"}), 400

        # Reset state
        similar_artists_update_state = {
            'status': 'running',
            'artists_processed': 0,
            'total_artists': 0,
            'current_artist': None,
            'error': None
        }

        # Start update in background
        similar_artists_executor.submit(_update_similar_artists_worker)

        return jsonify({"success": True, "message": "Similar artists update started"})

    except Exception as e:
        print(f"Error starting similar artists update: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/watchlist/similar-artists-status', methods=['GET'])
def get_similar_artists_update_status():
    """Get status of similar artists update"""
    try:
        global similar_artists_update_state
        return jsonify({"success": True, **similar_artists_update_state})
    except Exception as e:
        print(f"Error getting similar artists status: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/watchlist/artist/<artist_id>/config', methods=['GET', 'POST'])
def watchlist_artist_config(artist_id):
    """Get or update watchlist artist configuration"""
    try:
        from database.music_database import get_database

        database = get_database()

        if request.method == 'GET':
            # Get current config from database
            conn = sqlite3.connect(str(database.database_path))
            cursor = conn.cursor()
            cursor.execute("""
                SELECT include_albums, include_eps, include_singles,
                       include_live, include_remixes, include_acoustic, include_compilations,
                       artist_name, image_url, spotify_artist_id, itunes_artist_id,
                       last_scan_timestamp, date_added
                FROM watchlist_artists
                WHERE spotify_artist_id = ? OR itunes_artist_id = ?
            """, (artist_id, artist_id))
            result = cursor.fetchone()
            conn.close()

            if not result:
                return jsonify({"success": False, "error": "Artist not found in watchlist"}), 404
            
            # Determine if this is an iTunes or Spotify artist
            is_itunes_artist = artist_id.isdigit()
            spotify_id = result[9]  # spotify_artist_id from query
            itunes_id = result[10]  # itunes_artist_id from query

            # Get artist info from Spotify (only for Spotify artists)
            artist_info = None
            if not is_itunes_artist and spotify_client and spotify_client.is_authenticated() and spotify_id:
                try:
                    artist_data = spotify_client.sp.artist(spotify_id)
                    if artist_data:
                        artist_info = {
                            'id': artist_data['id'],
                            'name': artist_data['name'],
                            'image_url': artist_data['images'][0]['url'] if artist_data.get('images') else None,
                            'followers': artist_data.get('followers', {}).get('total', 0),
                            'popularity': artist_data.get('popularity', 0),
                            'genres': artist_data.get('genres', [])
                        }
                except Exception as e:
                    print(f"Warning: Could not fetch artist info from Spotify: {e}")

            # Fallback to database info if Spotify fetch failed
            if not artist_info:
                artist_info = {
                    'id': artist_id,
                    'name': result[7],  # artist_name
                    'image_url': result[8],  # image_url
                    'followers': 0,
                    'popularity': 0,
                    'genres': []
                }

            # Enrich with library artist data (banner, bio, style, mood, label)
            try:
                conn2 = sqlite3.connect(str(database.database_path))
                cur2 = conn2.cursor()
                cur2.execute("""
                    SELECT banner_url, summary, style, mood, label, genres
                    FROM artists
                    WHERE spotify_artist_id = ? OR itunes_artist_id = ?
                    LIMIT 1
                """, (artist_id, artist_id))
                lib_row = cur2.fetchone()
                if lib_row:
                    artist_info['banner_url'] = lib_row[0]
                    artist_info['summary'] = lib_row[1]
                    artist_info['style'] = lib_row[2]
                    artist_info['mood'] = lib_row[3]
                    artist_info['label'] = lib_row[4]
                    # Backfill genres from library if Spotify didn't provide any
                    if not artist_info.get('genres') and lib_row[5]:
                        try:
                            artist_info['genres'] = json.loads(lib_row[5])
                        except (json.JSONDecodeError, TypeError):
                            pass

                # Get recent releases for this watchlist artist
                cur2.execute("""
                    SELECT rr.album_name, rr.release_date, rr.album_cover_url, rr.track_count
                    FROM recent_releases rr
                    JOIN watchlist_artists wa ON rr.watchlist_artist_id = wa.id
                    WHERE wa.spotify_artist_id = ? OR wa.itunes_artist_id = ?
                    ORDER BY rr.release_date DESC
                    LIMIT 6
                """, (artist_id, artist_id))
                releases = [
                    {
                        'album_name': r[0],
                        'release_date': r[1],
                        'album_cover_url': r[2],
                        'track_count': r[3],
                    }
                    for r in cur2.fetchall()
                ]
                conn2.close()
            except Exception as e:
                print(f"Warning: Could not enrich artist from library: {e}")
                releases = []

            config = {
                'include_albums': bool(result[0]),  # Convert INTEGER to boolean
                'include_eps': bool(result[1]),
                'include_singles': bool(result[2]),
                'include_live': bool(result[3]),
                'include_remixes': bool(result[4]),
                'include_acoustic': bool(result[5]),
                'include_compilations': bool(result[6]),
                'last_scan_timestamp': result[11],
                'date_added': result[12],
            }

            return jsonify({
                "success": True,
                "config": config,
                "artist": artist_info,
                "recent_releases": releases
            })

        else:  # POST
            data = request.get_json()
            if not data:
                return jsonify({"success": False, "error": "No data provided"}), 400

            include_albums = data.get('include_albums', True)
            include_eps = data.get('include_eps', True)
            include_singles = data.get('include_singles', True)
            include_live = data.get('include_live', False)
            include_remixes = data.get('include_remixes', False)
            include_acoustic = data.get('include_acoustic', False)
            include_compilations = data.get('include_compilations', False)

            # Validate at least one release type is selected
            if not (include_albums or include_eps or include_singles):
                return jsonify({"success": False, "error": "At least one release type must be selected"}), 400

            # Update database
            conn = sqlite3.connect(str(database.database_path))
            cursor = conn.cursor()
            cursor.execute("""
                UPDATE watchlist_artists
                SET include_albums = ?, include_eps = ?, include_singles = ?,
                    include_live = ?, include_remixes = ?, include_acoustic = ?, include_compilations = ?,
                    updated_at = CURRENT_TIMESTAMP
                WHERE spotify_artist_id = ? OR itunes_artist_id = ?
            """, (int(include_albums), int(include_eps), int(include_singles),
                  int(include_live), int(include_remixes), int(include_acoustic), int(include_compilations),
                  artist_id, artist_id))
            conn.commit()

            if cursor.rowcount == 0:
                conn.close()
                return jsonify({"success": False, "error": "Artist not found in watchlist"}), 404

            conn.close()

            print(f"✅ Updated watchlist config for artist {artist_id}: albums={include_albums}, eps={include_eps}, singles={include_singles}, live={include_live}, remixes={include_remixes}, acoustic={include_acoustic}, compilations={include_compilations}")

            return jsonify({
                "success": True,
                "message": "Artist configuration updated successfully",
                "config": {
                    'include_albums': include_albums,
                    'include_eps': include_eps,
                    'include_singles': include_singles,
                    'include_live': include_live,
                    'include_remixes': include_remixes,
                    'include_acoustic': include_acoustic,
                    'include_compilations': include_compilations
                }
            })

    except Exception as e:
        print(f"Error in watchlist artist config: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/watchlist/global-config', methods=['GET', 'POST'])
def watchlist_global_config():
    """Get or update global watchlist configuration (overrides per-artist settings)"""
    try:
        if request.method == 'GET':
            config = {
                'global_override_enabled': config_manager.get('watchlist.global_override_enabled', False),
                'include_albums': config_manager.get('watchlist.global_include_albums', True),
                'include_eps': config_manager.get('watchlist.global_include_eps', True),
                'include_singles': config_manager.get('watchlist.global_include_singles', True),
                'include_live': config_manager.get('watchlist.global_include_live', False),
                'include_remixes': config_manager.get('watchlist.global_include_remixes', False),
                'include_acoustic': config_manager.get('watchlist.global_include_acoustic', False),
                'include_compilations': config_manager.get('watchlist.global_include_compilations', False),
            }
            return jsonify({"success": True, "config": config})

        else:  # POST
            data = request.get_json()
            if not data:
                return jsonify({"success": False, "error": "No data provided"}), 400

            global_override_enabled = data.get('global_override_enabled', False)
            include_albums = data.get('include_albums', True)
            include_eps = data.get('include_eps', True)
            include_singles = data.get('include_singles', True)
            include_live = data.get('include_live', False)
            include_remixes = data.get('include_remixes', False)
            include_acoustic = data.get('include_acoustic', False)
            include_compilations = data.get('include_compilations', False)

            # When override is enabled, validate at least one release type
            if global_override_enabled and not (include_albums or include_eps or include_singles):
                return jsonify({"success": False, "error": "At least one release type must be selected"}), 400

            config_manager.set('watchlist.global_override_enabled', global_override_enabled)
            config_manager.set('watchlist.global_include_albums', include_albums)
            config_manager.set('watchlist.global_include_eps', include_eps)
            config_manager.set('watchlist.global_include_singles', include_singles)
            config_manager.set('watchlist.global_include_live', include_live)
            config_manager.set('watchlist.global_include_remixes', include_remixes)
            config_manager.set('watchlist.global_include_acoustic', include_acoustic)
            config_manager.set('watchlist.global_include_compilations', include_compilations)

            print(f"✅ Updated global watchlist config: override={global_override_enabled}, "
                  f"albums={include_albums}, eps={include_eps}, singles={include_singles}, "
                  f"live={include_live}, remixes={include_remixes}, acoustic={include_acoustic}, "
                  f"compilations={include_compilations}")

            return jsonify({
                "success": True,
                "message": "Global watchlist configuration updated",
                "config": {
                    'global_override_enabled': global_override_enabled,
                    'include_albums': include_albums,
                    'include_eps': include_eps,
                    'include_singles': include_singles,
                    'include_live': include_live,
                    'include_remixes': include_remixes,
                    'include_acoustic': include_acoustic,
                    'include_compilations': include_compilations,
                }
            })

    except Exception as e:
        print(f"Error in watchlist global config: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

def _apply_watchlist_global_overrides(watchlist_artists):
    """If global override is enabled, overwrite per-artist settings on WatchlistArtist objects."""
    if not config_manager.get('watchlist.global_override_enabled', False):
        return
    # Read global settings once
    g_albums = config_manager.get('watchlist.global_include_albums', True)
    g_eps = config_manager.get('watchlist.global_include_eps', True)
    g_singles = config_manager.get('watchlist.global_include_singles', True)
    g_live = config_manager.get('watchlist.global_include_live', False)
    g_remixes = config_manager.get('watchlist.global_include_remixes', False)
    g_acoustic = config_manager.get('watchlist.global_include_acoustic', False)
    g_compilations = config_manager.get('watchlist.global_include_compilations', False)
    print(f"🌐 [Watchlist] Global override is ACTIVE — applying to {len(watchlist_artists)} artists "
          f"(albums={g_albums}, eps={g_eps}, singles={g_singles}, live={g_live}, "
          f"remixes={g_remixes}, acoustic={g_acoustic}, compilations={g_compilations})")
    for artist in watchlist_artists:
        artist.include_albums = g_albums
        artist.include_eps = g_eps
        artist.include_singles = g_singles
        artist.include_live = g_live
        artist.include_remixes = g_remixes
        artist.include_acoustic = g_acoustic
        artist.include_compilations = g_compilations

def _update_similar_artists_worker():
    """Background worker to update similar artists for all watchlist artists"""
    global similar_artists_update_state

    try:
        from core.watchlist_scanner import get_watchlist_scanner
        from database.music_database import get_database
        import time

        print("🎵 [Similar Artists] Starting similar artists update...")

        database = get_database()
        all_profiles = database.get_all_profiles()

        # Build per-profile artist lists and deduplicate for API calls
        # artist_key -> (artist_obj, [profile_ids])
        artist_profiles = {}
        for p in all_profiles:
            for artist in database.get_watchlist_artists(profile_id=p['id']):
                key = (artist.spotify_artist_id or '', artist.itunes_artist_id or '', artist.artist_name.lower())
                if key not in artist_profiles:
                    artist_profiles[key] = (artist, [])
                artist_profiles[key][1].append(p['id'])

        if not artist_profiles:
            similar_artists_update_state['status'] = 'completed'
            print("📭 [Similar Artists] No watchlist artists to process")
            return

        similar_artists_update_state['total_artists'] = len(artist_profiles)
        print(f"📊 [Similar Artists] Processing {len(artist_profiles)} unique watchlist artists across {len(all_profiles)} profiles")

        scanner = get_watchlist_scanner(spotify_client)

        for idx, (key, (artist, profile_ids)) in enumerate(artist_profiles.items(), 1):
            try:
                similar_artists_update_state['artists_processed'] = idx
                similar_artists_update_state['current_artist'] = artist.artist_name

                print(f"[{idx}/{len(artist_profiles)}] Updating similar artists for {artist.artist_name} (profiles: {profile_ids})")

                # Update similar artists for each profile that watches this artist
                for pid in profile_ids:
                    scanner.update_similar_artists(artist, limit=10, profile_id=pid)

                # Rate limiting
                if idx < len(artist_profiles):
                    time.sleep(2.0)  # 2 seconds between artists

            except Exception as artist_error:
                print(f"❌ [Similar Artists] Error processing {artist.artist_name}: {artist_error}")
                continue

        # Update complete
        similar_artists_update_state['status'] = 'completed'
        similar_artists_update_state['current_artist'] = None

        print(f"✅ [Similar Artists] Update complete! Processed {len(artist_profiles)} artists")

    except Exception as e:
        print(f"❌ [Similar Artists] Critical error: {e}")
        import traceback
        traceback.print_exc()

        similar_artists_update_state['status'] = 'error'
        similar_artists_update_state['error'] = str(e)

# --- Watchlist Auto-Scanning System ---

watchlist_scan_state = {
    'status': 'idle',
    'results': [],
    'summary': {},
    'error': None
}

def _process_watchlist_scan_automatically(automation_id=None):
    """Main automatic scanning logic that runs in background thread."""
    global watchlist_auto_scanning, watchlist_auto_scanning_timestamp, watchlist_scan_state

    print("🤖 [Auto-Watchlist] Timer triggered - starting automatic watchlist scan...")

    _enrichment_was_running = False
    _itunes_enrichment_was_running = False

    try:
        # CRITICAL FIX: Use smart stuck detection BEFORE acquiring lock
        # This prevents deadlock and handles stuck flags (2-hour timeout)
        if is_watchlist_actually_scanning():
            print("⚠️ [Auto-Watchlist] Already scanning (verified with stuck detection), skipping.")
            return

        with watchlist_timer_lock:
            # Re-check inside lock to handle race conditions
            if watchlist_auto_scanning:
                print("⚠️ [Auto-Watchlist] Already scanning (race condition check), skipping.")
                return

            # Set flag and timestamp
            import time
            watchlist_auto_scanning = True
            watchlist_auto_scanning_timestamp = time.time()
            print(f"🔒 [Auto-Watchlist] Flag set at timestamp {watchlist_auto_scanning_timestamp}")

        # Use app context for database operations
        with app.app_context():
            from core.watchlist_scanner import get_watchlist_scanner
            from database.music_database import get_database

            # Check if we have artists to scan across all profiles
            database = get_database()
            # Auto-scan covers all profiles
            all_profiles = database.get_all_profiles()
            watchlist_count = sum(database.get_watchlist_count(profile_id=p['id']) for p in all_profiles)
            print(f"🔍 [Auto-Watchlist] Watchlist count check: {watchlist_count} artists found across {len(all_profiles)} profiles")

            if watchlist_count == 0:
                print("ℹ️ [Auto-Watchlist] No artists in watchlist for auto-scanning.")
                with watchlist_timer_lock:
                    watchlist_auto_scanning = False
                    watchlist_auto_scanning_timestamp = 0
                return

            if not spotify_client or not spotify_client.is_authenticated():
                print("ℹ️ [Auto-Watchlist] Spotify client not available or not authenticated.")
                with watchlist_timer_lock:
                    watchlist_auto_scanning = False
                    watchlist_auto_scanning_timestamp = 0
                return

            print(f"👁️ [Auto-Watchlist] Found {watchlist_count} artists in watchlist, starting automatic scan...")
            _update_automation_progress(automation_id, progress=5, phase='Loading watchlist',
                                         log_line=f'{watchlist_count} artists across {len(all_profiles)} profiles', log_type='info')

            # Get list of artists to scan (all profiles combined for auto-scan)
            watchlist_artists = []
            for p in all_profiles:
                watchlist_artists.extend(database.get_watchlist_artists(profile_id=p['id']))
            scanner = get_watchlist_scanner(spotify_client)

            # Apply global overrides if enabled
            _apply_watchlist_global_overrides(watchlist_artists)

            # Initialize detailed progress tracking (same as manual scan)
            watchlist_scan_state = {
                'status': 'scanning',
                'started_at': datetime.now(),
                'total_artists': len(watchlist_artists),
                'current_artist_index': 0,
                'current_artist_name': '',
                'current_artist_image_url': '',
                'current_phase': 'starting',
                'albums_to_check': 0,
                'albums_checked': 0,
                'current_album': '',
                'current_album_image_url': '',
                'current_track_name': '',
                'tracks_found_this_scan': 0,
                'tracks_added_this_scan': 0,
                'recent_wishlist_additions': [],
                'results': [],
                'summary': {},
                'error': None
            }

            scan_results = []

            # Pause enrichment workers during scan to reduce API contention
            if spotify_enrichment_worker and not spotify_enrichment_worker.paused:
                spotify_enrichment_worker.pause()
                _enrichment_was_running = True
                print("⏸️ [Auto-Watchlist] Paused Spotify enrichment worker during scan")
            if itunes_enrichment_worker and not itunes_enrichment_worker.paused:
                itunes_enrichment_worker.pause()
                _itunes_enrichment_was_running = True
                print("⏸️ [Auto-Watchlist] Paused iTunes enrichment worker during scan")

            # Dynamic delay calculation based on scan scope
            lookback_period = scanner._get_lookback_period_setting()
            is_full_discography = (lookback_period == 'all')
            artist_count = len(watchlist_artists)

            base_artist_delay = 2.0
            base_album_delay = 0.5

            # Scale up for full discography (way more albums per artist)
            if is_full_discography:
                base_artist_delay *= 2.0
                base_album_delay *= 2.0

            # Scale up further for large artist counts (sustained API pressure)
            if artist_count > 200:
                base_artist_delay *= 1.5
                base_album_delay *= 1.25
            elif artist_count > 100:
                base_artist_delay *= 1.25

            artist_delay = base_artist_delay
            album_delay = base_album_delay
            print(f"📊 [Auto-Watchlist] Scan parameters: {artist_count} artists, lookback={lookback_period}, "
                  f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album")

            # Circuit breaker: pause scan on consecutive rate-limit failures
            consecutive_failures = 0
            CIRCUIT_BREAKER_THRESHOLD = 3
            circuit_breaker_pause = 60  # seconds, doubles each trigger, max 600s

            # Scan each artist with detailed tracking
            for i, artist in enumerate(watchlist_artists):
                try:
                    # Fetch artist image using provider-aware method
                    artist_image_url = scanner.get_artist_image_url(artist) or ''

                    pct = 5 + (i / max(1, len(watchlist_artists))) * 90
                    _update_automation_progress(automation_id, progress=pct,
                        phase=f'Scanning: {artist.artist_name} ({i+1}/{len(watchlist_artists)})',
                        current_item=artist.artist_name,
                        processed=i, total=len(watchlist_artists))

                    # Update progress
                    watchlist_scan_state.update({
                        'current_artist_index': i + 1,
                        'current_artist_name': artist.artist_name,
                        'current_artist_image_url': artist_image_url,
                        'current_phase': 'fetching_discography',
                        'albums_to_check': 0,
                        'albums_checked': 0,
                        'current_album': '',
                        'current_album_image_url': '',
                        'current_track_name': ''
                    })

                    # Get artist discography using provider-aware method
                    albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp)

                    if albums is None:
                        scan_results.append(type('ScanResult', (), {
                            'artist_name': artist.artist_name,
                            'spotify_artist_id': artist.spotify_artist_id,
                            'albums_checked': 0,
                            'new_tracks_found': 0,
                            'tracks_added_to_wishlist': 0,
                            'success': False,
                            'error_message': "Failed to get artist discography"
                        })())
                        continue

                    # Update with album count
                    watchlist_scan_state.update({
                        'current_phase': 'checking_albums',
                        'albums_to_check': len(albums),
                        'albums_checked': 0
                    })

                    # Track progress for this artist
                    artist_new_tracks = 0
                    artist_added_tracks = 0

                    # Scan each album
                    for album_index, album in enumerate(albums):
                        try:
                            # Get album tracks using provider-aware method
                            album_data = scanner.metadata_service.get_album(album.id)
                            if not album_data or 'tracks' not in album_data:
                                continue

                            tracks = album_data['tracks']['items']

                            # Check release type filter (album/EP/single)
                            if not scanner._should_include_release(len(tracks), artist):
                                continue

                            # Get album image
                            album_image_url = ''
                            if 'images' in album_data and album_data['images']:
                                album_image_url = album_data['images'][0]['url']

                            watchlist_scan_state.update({
                                'albums_checked': album_index + 1,
                                'current_album': album.name,
                                'current_album_image_url': album_image_url,
                                'current_phase': f'checking_album_{album_index + 1}_of_{len(albums)}'
                            })

                            # Check each track
                            for track in tracks:
                                # Check content type filter (live/remix/acoustic/compilation)
                                if not scanner._should_include_track(track, album_data, artist):
                                    continue

                                # Update current track being processed
                                track_name = track.get('name', 'Unknown Track')
                                watchlist_scan_state['current_track_name'] = track_name

                                if scanner.is_track_missing_from_library(track):
                                    artist_new_tracks += 1
                                    watchlist_scan_state['tracks_found_this_scan'] += 1

                                    # Add to wishlist
                                    if scanner.add_track_to_wishlist(track, album_data, artist):
                                        artist_added_tracks += 1
                                        watchlist_scan_state['tracks_added_this_scan'] += 1

                                        # Add to recent wishlist additions feed
                                        track_artists = track.get('artists', [])
                                        track_artist_name = track_artists[0].get('name', 'Unknown Artist') if track_artists else 'Unknown Artist'

                                        watchlist_scan_state['recent_wishlist_additions'].insert(0, {
                                            'track_name': track_name,
                                            'artist_name': track_artist_name,
                                            'album_image_url': album_image_url
                                        })

                                        # Keep only last 10
                                        if len(watchlist_scan_state['recent_wishlist_additions']) > 10:
                                            watchlist_scan_state['recent_wishlist_additions'].pop()

                            # Rate-limited delay between albums
                            import time
                            time.sleep(album_delay)

                        except Exception as e:
                            print(f"Error checking album {album.name}: {e}")
                            continue

                    # Update scan timestamp
                    scanner.update_artist_scan_timestamp(artist)

                    # Store result
                    scan_results.append(type('ScanResult', (), {
                        'artist_name': artist.artist_name,
                        'spotify_artist_id': artist.spotify_artist_id,
                        'albums_checked': len(albums),
                        'new_tracks_found': artist_new_tracks,
                        'tracks_added_to_wishlist': artist_added_tracks,
                        'success': True,
                        'error_message': None
                    })())

                    print(f"✅ Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist")
                    if artist_new_tracks > 0:
                        _update_automation_progress(automation_id,
                            log_line=f'{artist.artist_name} — {artist_new_tracks} new, {artist_added_tracks} added', log_type='success')
                    else:
                        _update_automation_progress(automation_id,
                            log_line=f'{artist.artist_name} — no new tracks', log_type='skip')

                    # Emit watchlist_new_release event if new tracks were found
                    if artist_new_tracks > 0:
                        try:
                            if automation_engine:
                                automation_engine.emit('watchlist_new_release', {
                                    'artist': artist.artist_name,
                                    'new_tracks': str(artist_new_tracks),
                                    'added_to_wishlist': str(artist_added_tracks),
                                })
                        except Exception:
                            pass

                    # Delay between artists
                    if i < len(watchlist_artists) - 1:
                        watchlist_scan_state['current_phase'] = 'rate_limiting'
                        time.sleep(artist_delay)

                    # Reset circuit breaker on successful artist scan
                    consecutive_failures = 0
                    circuit_breaker_pause = 60

                except Exception as e:
                    print(f"Error scanning artist {artist.artist_name}: {e}")
                    _update_automation_progress(automation_id,
                        log_line=f'{artist.artist_name} — error: {str(e)[:60]}', log_type='error')

                    # Circuit breaker: detect consecutive rate-limit failures
                    error_str = str(e).lower()
                    if "429" in error_str or "rate limit" in error_str:
                        consecutive_failures += 1
                        if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD:
                            print(f"🛑 [Auto-Watchlist] Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s")
                            watchlist_scan_state['current_phase'] = 'circuit_breaker_pause'
                            time.sleep(circuit_breaker_pause)
                            circuit_breaker_pause = min(circuit_breaker_pause * 2, 600)
                            consecutive_failures = 0
                    else:
                        consecutive_failures = 0

                    scan_results.append(type('ScanResult', (), {
                        'artist_name': artist.artist_name,
                        'spotify_artist_id': artist.spotify_artist_id,
                        'albums_checked': 0,
                        'new_tracks_found': 0,
                        'tracks_added_to_wishlist': 0,
                        'success': False,
                        'error_message': str(e)
                    })())
                    continue

            # Update state with results
            successful_scans = [r for r in scan_results if r.success]
            total_new_tracks = sum(r.new_tracks_found for r in successful_scans)
            total_added_to_wishlist = sum(r.tracks_added_to_wishlist for r in successful_scans)

            watchlist_scan_state['status'] = 'completed'
            watchlist_scan_state['results'] = scan_results
            watchlist_scan_state['completed_at'] = datetime.now()
            watchlist_scan_state['summary'] = {
                'total_artists': len(scan_results),
                'successful_scans': len(successful_scans),
                'new_tracks_found': total_new_tracks,
                'tracks_added_to_wishlist': total_added_to_wishlist
            }

            print(f"Automatic watchlist scan completed: {len(successful_scans)}/{len(scan_results)} artists scanned successfully")
            print(f"Found {total_new_tracks} new tracks, added {total_added_to_wishlist} to wishlist")
            _update_automation_progress(automation_id, progress=95, phase='Scan complete',
                                         log_line=f'Scanned {len(successful_scans)} artists — {total_new_tracks} new tracks, {total_added_to_wishlist} added to wishlist',
                                         log_type='success' if total_new_tracks > 0 else 'info')

            # Populate discovery pool from similar artists (per-profile)
            print("🎵 Starting discovery pool population...")
            watchlist_scan_state['current_phase'] = 'populating_discovery_pool'
            try:
                for p in all_profiles:
                    scanner.populate_discovery_pool(profile_id=p['id'])
                print("✅ Discovery pool population complete")
            except Exception as discovery_error:
                print(f"⚠️ Error populating discovery pool: {discovery_error}")
                import traceback
                traceback.print_exc()

            # Update ListenBrainz playlists cache
            print("🧠 Starting ListenBrainz playlists update...")
            watchlist_scan_state['current_phase'] = 'updating_listenbrainz'
            try:
                from core.listenbrainz_manager import ListenBrainzManager
                lb_manager = ListenBrainzManager(str(get_database().database_path))
                lb_result = lb_manager.update_all_playlists()
                if lb_result.get('success'):
                    summary = lb_result.get('summary', {})
                    print(f"✅ ListenBrainz update complete: {summary}")
                else:
                    print(f"⚠️ ListenBrainz update had issues: {lb_result.get('error', 'Unknown error')}")
            except Exception as lb_error:
                print(f"⚠️ Error updating ListenBrainz: {lb_error}")
                import traceback
                traceback.print_exc()

            # Update current seasonal playlist (weekly refresh)
            print("🎃 Starting seasonal content update...")
            watchlist_scan_state['current_phase'] = 'updating_seasonal'
            try:
                from core.seasonal_discovery import get_seasonal_discovery_service
                seasonal_service = get_seasonal_discovery_service(spotify_client, database)

                # Only update the current active season
                current_season = seasonal_service.get_current_season()
                if current_season:
                    if seasonal_service.should_populate_seasonal_content(current_season, days_threshold=7):
                        print(f"🎃 Updating {current_season} seasonal content...")
                        seasonal_service.populate_seasonal_content(current_season)
                        seasonal_service.curate_seasonal_playlist(current_season)
                        print(f"✅ {current_season.capitalize()} seasonal content updated")
                    else:
                        print(f"⏭️ {current_season.capitalize()} seasonal content recently updated, skipping")
                else:
                    print("ℹ️ No active season at this time")
            except Exception as seasonal_error:
                print(f"⚠️ Error updating seasonal content: {seasonal_error}")
                import traceback
                traceback.print_exc()

            # Add activity for watchlist scan completion
            if total_added_to_wishlist > 0:
                add_activity_item("👁️", "Watchlist Scan Complete", f"{total_added_to_wishlist} new tracks added to wishlist", "Now")

            try:
                if automation_engine:
                    automation_engine.emit('watchlist_scan_completed', {
                        'artists_scanned': str(len(scan_results)),
                        'new_tracks_found': str(total_new_tracks),
                        'tracks_added': str(total_added_to_wishlist),
                    })
            except Exception:
                pass

    except Exception as e:
        print(f"❌ Error in automatic watchlist scan: {e}")
        import traceback
        traceback.print_exc()
        _update_automation_progress(automation_id, log_line=f'Error: {str(e)}', log_type='error')

        watchlist_scan_state['status'] = 'error'
        watchlist_scan_state['error'] = str(e)
        raise  # re-raise so automation wrapper returns error status

    finally:
        # Resume enrichment workers if we paused them
        if _enrichment_was_running and spotify_enrichment_worker:
            spotify_enrichment_worker.resume()
            print("▶️ [Auto-Watchlist] Resumed Spotify enrichment worker after scan")
        if _itunes_enrichment_was_running and itunes_enrichment_worker:
            itunes_enrichment_worker.resume()
            print("▶️ [Auto-Watchlist] Resumed iTunes enrichment worker after scan")

        # Always reset flag
        with watchlist_timer_lock:
            watchlist_auto_scanning = False
            watchlist_auto_scanning_timestamp = 0

    print("✅ Automatic watchlist scanning complete")

# --- Metadata Updater System ---

from concurrent.futures import ThreadPoolExecutor, as_completed

# Global state for metadata update process
metadata_update_state = {
    'status': 'idle',
    'current_artist': '',
    'processed': 0,
    'total': 0,
    'percentage': 0.0,
    'successful': 0,
    'failed': 0,
    'started_at': None,
    'completed_at': None,
    'error': None,
    'refresh_interval_days': 30
}

metadata_update_worker = None
metadata_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="metadata_update")

# ===============================
# == DISCOVER PAGE ENDPOINTS   ==
# ===============================

def _get_active_discovery_source():
    """
    Determine which music source is active for discovery.
    Returns 'spotify' if Spotify is authenticated, 'itunes' otherwise.
    """
    if spotify_client and spotify_client.is_spotify_authenticated():
        return 'spotify'
    return 'itunes'


@app.route('/api/discover/hero', methods=['GET'])
def get_discover_hero():
    """Get featured similar artists for hero slideshow"""
    try:
        database = get_database()

        # Determine active source
        active_source = _get_active_discovery_source()
        print(f"🎵 Discover hero using source: {active_source}")

        # Import iTunes client for fallback
        from core.itunes_client import iTunesClient
        itunes_client = iTunesClient()

        # Get top similar artists (excluding watchlist, cycled by last_featured)
        pid = get_current_profile_id()
        similar_artists = database.get_top_similar_artists(limit=50, profile_id=pid)

        # FALLBACK: If no similar artists exist, use watchlist artists for Hero section
        if not similar_artists:
            print("[Discover Hero] No similar artists found, falling back to watchlist artists")
            watchlist_artists = database.get_watchlist_artists(profile_id=pid)

            if not watchlist_artists:
                return jsonify({"success": True, "artists": [], "source": active_source})

            # Convert watchlist artists to hero format
            import random
            shuffled_watchlist = list(watchlist_artists)
            random.shuffle(shuffled_watchlist)

            hero_artists = []
            for artist in shuffled_watchlist[:10]:
                artist_id = artist.itunes_artist_id if active_source == 'itunes' else artist.spotify_artist_id
                if not artist_id:
                    continue

                artist_data = {
                    "spotify_artist_id": artist.spotify_artist_id,
                    "itunes_artist_id": artist.itunes_artist_id,
                    "artist_id": artist_id,
                    "artist_name": artist.artist_name,
                    "occurrence_count": 1,
                    "similarity_rank": 1,
                    "source": active_source,
                    "is_watchlist": True
                }

                # Try to get artist image
                try:
                    if active_source == 'itunes' and artist.itunes_artist_id:
                        itunes_artist = itunes_client.get_artist(artist.itunes_artist_id)
                        if itunes_artist:
                            # Use canonical name from iTunes API (normalized to 'name' field)
                            artist_data['artist_name'] = itunes_artist.get('name', artist.artist_name)
                            artist_data['image_url'] = itunes_artist.get('images', [{}])[0].get('url') if itunes_artist.get('images') else None
                            artist_data['genres'] = itunes_artist.get('genres', [])
                    elif active_source == 'spotify' and artist.spotify_artist_id:
                        if spotify_client and spotify_client.is_authenticated():
                            sp_artist = spotify_client.get_artist(artist.spotify_artist_id)
                            if sp_artist and sp_artist.get('images'):
                                # Use canonical name from Spotify API
                                artist_data['artist_name'] = sp_artist.get('name', artist.artist_name)
                                artist_data['image_url'] = sp_artist['images'][0]['url'] if sp_artist['images'] else None
                                artist_data['genres'] = sp_artist.get('genres', [])
                except Exception as img_err:
                    print(f"Could not fetch watchlist artist image: {img_err}")

                hero_artists.append(artist_data)

            print(f"[Discover Hero] Returning {len(hero_artists)} watchlist artists as fallback")
            return jsonify({"success": True, "artists": hero_artists, "source": active_source, "fallback": "watchlist"})

        # Filter to artists that have the appropriate ID for the active source
        valid_artists = []
        for artist in similar_artists:
            if active_source == 'spotify' and artist.similar_artist_spotify_id:
                valid_artists.append(artist)
            elif active_source == 'itunes' and artist.similar_artist_itunes_id:
                valid_artists.append(artist)
            # If we have both IDs, include regardless of source
            elif artist.similar_artist_spotify_id and artist.similar_artist_itunes_id:
                valid_artists.append(artist)

        # FALLBACK: If no valid artists for iTunes, try to resolve iTunes IDs on-the-fly
        if active_source == 'itunes' and not valid_artists:
            print(f"[iTunes Fallback] No artists with iTunes IDs found, attempting on-the-fly resolution for {len(similar_artists)} artists")
            resolved_count = 0
            for artist in similar_artists:
                if artist.similar_artist_itunes_id:
                    valid_artists.append(artist)
                    continue
                # Try to resolve iTunes ID by name
                try:
                    itunes_results = itunes_client.search_artists(artist.similar_artist_name, limit=1)
                    if itunes_results and len(itunes_results) > 0:
                        itunes_id = itunes_results[0].id
                        # Cache the resolved ID for future use
                        database.update_similar_artist_itunes_id(artist.id, itunes_id)
                        # Create a modified artist object with the resolved ID
                        artist.similar_artist_itunes_id = itunes_id
                        valid_artists.append(artist)
                        resolved_count += 1
                        print(f"  [Resolved] {artist.similar_artist_name} -> iTunes ID: {itunes_id}")
                except Exception as resolve_err:
                    print(f"  [Failed] Could not resolve iTunes ID for {artist.similar_artist_name}: {resolve_err}")
                # Stop after 10 successful resolutions to avoid rate limiting
                if len(valid_artists) >= 10:
                    break
            print(f"[iTunes Fallback] Resolved {resolved_count} artists with iTunes IDs")

        print(f"[Discover Hero] Found {len(valid_artists)} valid artists for source: {active_source}")

        # Take top 10 (already ordered by least-recently-featured, then quality)
        similar_artists = valid_artists[:10]

        # Convert to JSON format with data enrichment from appropriate source
        hero_artists = []
        for artist in similar_artists:
            # Use the ID for the active source, falling back to the other if needed
            if active_source == 'spotify':
                artist_id = artist.similar_artist_spotify_id or artist.similar_artist_itunes_id
            else:
                artist_id = artist.similar_artist_itunes_id or artist.similar_artist_spotify_id

            artist_data = {
                "spotify_artist_id": artist.similar_artist_spotify_id,
                "itunes_artist_id": artist.similar_artist_itunes_id,
                "artist_id": artist_id,  # The ID for the current active source
                "artist_name": artist.similar_artist_name,
                "occurrence_count": artist.occurrence_count,
                "similarity_rank": artist.similarity_rank,
                "source": active_source
            }

            # Try to get artist image from the active source
            try:
                if active_source == 'spotify' and artist.similar_artist_spotify_id:
                    if spotify_client and spotify_client.is_authenticated():
                        sp_artist = spotify_client.get_artist(artist.similar_artist_spotify_id)
                        if sp_artist and sp_artist.get('images'):
                            # Use canonical name from Spotify API to ensure it matches the image
                            artist_data['artist_name'] = sp_artist.get('name', artist.similar_artist_name)
                            artist_data['image_url'] = sp_artist['images'][0]['url'] if sp_artist['images'] else None
                            artist_data['genres'] = sp_artist.get('genres', [])
                            artist_data['popularity'] = sp_artist.get('popularity', 0)
                elif active_source == 'itunes' and artist.similar_artist_itunes_id:
                    itunes_artist = itunes_client.get_artist(artist.similar_artist_itunes_id)
                    if itunes_artist:
                        # iTunes client normalizes to Spotify format, so use 'name' not 'artistName'
                        artist_data['artist_name'] = itunes_artist.get('name', artist.similar_artist_name)
                        artist_data['image_url'] = itunes_artist.get('images', [{}])[0].get('url') if itunes_artist.get('images') else None
                        artist_data['genres'] = itunes_artist.get('genres', [])
                        artist_data['popularity'] = itunes_artist.get('popularity', 0)
            except Exception as img_err:
                print(f"Could not fetch artist image: {img_err}")

            hero_artists.append(artist_data)

        # Mark these artists as featured so they cycle to the back of the queue
        featured_names = [a["artist_name"] for a in hero_artists]
        database.mark_artists_featured(featured_names)

        return jsonify({"success": True, "artists": hero_artists, "source": active_source})

    except Exception as e:
        print(f"Error getting discover hero: {e}")
        return jsonify({"success": False, "error": str(e)}), 500


@app.route('/api/discover/similar-artists', methods=['GET'])
def get_discover_similar_artists():
    """Get all recommended similar artists (basic data, no enrichment for speed)"""
    try:
        database = get_database()
        active_source = _get_active_discovery_source()

        similar_artists = database.get_top_similar_artists(limit=200, profile_id=get_current_profile_id())

        if not similar_artists:
            return jsonify({"success": True, "artists": [], "source": active_source, "count": 0})

        # Filter to artists with valid ID for active source
        result_artists = []
        for artist in similar_artists:
            has_spotify = bool(artist.similar_artist_spotify_id)
            has_itunes = bool(artist.similar_artist_itunes_id)

            if active_source == 'spotify' and not has_spotify and not has_itunes:
                continue
            if active_source == 'itunes' and not has_itunes and not has_spotify:
                continue

            if active_source == 'spotify':
                artist_id = artist.similar_artist_spotify_id or artist.similar_artist_itunes_id
            else:
                artist_id = artist.similar_artist_itunes_id or artist.similar_artist_spotify_id

            result_artists.append({
                "artist_id": artist_id,
                "spotify_artist_id": artist.similar_artist_spotify_id,
                "itunes_artist_id": artist.similar_artist_itunes_id,
                "artist_name": artist.similar_artist_name,
                "occurrence_count": artist.occurrence_count,
                "similarity_rank": artist.similarity_rank,
                "source": active_source,
            })

        print(f"[Similar Artists] {len(similar_artists)} from DB, {len(result_artists)} valid for {active_source}")

        return jsonify({
            "success": True,
            "artists": result_artists,
            "source": active_source,
            "count": len(result_artists)
        })

    except Exception as e:
        print(f"Error getting similar artists: {e}")
        return jsonify({"success": False, "error": str(e)}), 500


@app.route('/api/discover/similar-artists/enrich', methods=['POST'])
def enrich_similar_artists():
    """Enrich a batch of artist IDs with images/genres from Spotify or iTunes"""
    try:
        data = request.get_json()
        artist_ids = data.get('artist_ids', [])
        source = data.get('source', 'spotify')

        if not artist_ids:
            return jsonify({"success": True, "artists": {}})

        enriched = {}

        if source == 'spotify' and spotify_client and spotify_client.is_authenticated():
            try:
                batch_result = spotify_client.sp.artists(artist_ids[:50])
                if batch_result and 'artists' in batch_result:
                    for sp_artist in batch_result['artists']:
                        if sp_artist:
                            enriched[sp_artist['id']] = {
                                "artist_name": sp_artist.get('name'),
                                "image_url": sp_artist['images'][0]['url'] if sp_artist.get('images') else None,
                                "genres": sp_artist.get('genres', [])[:3],
                                "popularity": sp_artist.get('popularity', 0)
                            }
            except Exception as e:
                print(f"Error enriching Spotify batch: {e}")
        else:
            from core.itunes_client import iTunesClient
            itunes_client = iTunesClient()
            for aid in artist_ids[:50]:
                try:
                    itunes_artist = itunes_client.get_artist(aid)
                    if itunes_artist:
                        enriched[aid] = {
                            "artist_name": itunes_artist.get('name'),
                            "image_url": itunes_artist.get('images', [{}])[0].get('url') if itunes_artist.get('images') else None,
                            "genres": itunes_artist.get('genres', [])[:3],
                            "popularity": 0
                        }
                except Exception:
                    pass

        return jsonify({"success": True, "artists": enriched})

    except Exception as e:
        print(f"Error enriching similar artists: {e}")
        return jsonify({"success": False, "error": str(e)}), 500


@app.route('/api/discover/recent-releases', methods=['GET'])
def get_discover_recent_releases():
    """Get cached recent albums from watchlist and similar artists"""
    try:
        database = get_database()

        # Determine active source
        active_source = _get_active_discovery_source()

        # Get cached recent albums filtered by source (max 20)
        albums = database.get_discovery_recent_albums(limit=20, source=active_source, profile_id=get_current_profile_id())

        return jsonify({"success": True, "albums": albums, "source": active_source})

    except Exception as e:
        print(f"Error getting recent releases: {e}")
        return jsonify({"success": False, "error": str(e)}), 500


@app.route('/api/discover/release-radar', methods=['GET'])
def get_discover_release_radar():
    """Get release radar playlist - curated selection that stays consistent until next update"""
    try:
        database = get_database()

        # Determine active source - release radar works with any source now
        active_source = _get_active_discovery_source()

        # Try source-specific playlist first, then fall back to generic
        pid = get_current_profile_id()
        curated_track_ids = database.get_curated_playlist(f'release_radar_{active_source}', profile_id=pid)
        if not curated_track_ids:
            curated_track_ids = database.get_curated_playlist('release_radar', profile_id=pid)

        if curated_track_ids:
            # Use curated selection - fetch track data from discovery pool filtered by source
            discovery_tracks = database.get_discovery_pool_tracks(limit=5000, new_releases_only=False, source=active_source, profile_id=pid)

            # Build lookup dict with source-appropriate IDs
            tracks_by_id = {}
            for track in discovery_tracks:
                if active_source == 'spotify' and track.spotify_track_id:
                    tracks_by_id[track.spotify_track_id] = track
                elif active_source == 'itunes' and track.itunes_track_id:
                    tracks_by_id[track.itunes_track_id] = track

            selected_tracks = []
            for track_id in curated_track_ids:
                if track_id in tracks_by_id:
                    track = tracks_by_id[track_id]

                    # Parse track_data_json if it's a string
                    track_data = track.track_data_json
                    if isinstance(track_data, str):
                        try:
                            track_data = json.loads(track_data)
                        except:
                            track_data = None

                    selected_tracks.append({
                        "track_id": track.spotify_track_id or track.itunes_track_id,
                        "spotify_track_id": track.spotify_track_id,
                        "itunes_track_id": track.itunes_track_id,
                        "track_name": track.track_name,
                        "artist_name": track.artist_name,
                        "album_name": track.album_name,
                        "album_cover_url": track.album_cover_url,
                        "duration_ms": track.duration_ms,
                        "track_data_json": track_data,
                        "source": track.source
                    })

            return jsonify({"success": True, "tracks": selected_tracks, "source": active_source})

        # Fallback: no curated playlist exists (shouldn't happen after first scan)
        return jsonify({"success": True, "tracks": [], "source": active_source})

    except Exception as e:
        print(f"Error getting release radar: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/weekly', methods=['GET'])
def get_discover_weekly():
    """Get discovery weekly playlist - curated selection that stays consistent until next update"""
    try:
        database = get_database()

        # Determine active source
        active_source = _get_active_discovery_source()

        # Try source-specific playlist first, then fall back to generic
        pid = get_current_profile_id()
        curated_track_ids = database.get_curated_playlist(f'discovery_weekly_{active_source}', profile_id=pid)
        if not curated_track_ids:
            curated_track_ids = database.get_curated_playlist('discovery_weekly', profile_id=pid)

        if curated_track_ids:
            # Use curated selection - fetch track data from discovery pool filtered by source
            discovery_tracks = database.get_discovery_pool_tracks(limit=5000, new_releases_only=False, source=active_source, profile_id=pid)

            # Build lookup dict with source-appropriate IDs
            tracks_by_id = {}
            for track in discovery_tracks:
                if active_source == 'spotify' and track.spotify_track_id:
                    tracks_by_id[track.spotify_track_id] = track
                elif active_source == 'itunes' and track.itunes_track_id:
                    tracks_by_id[track.itunes_track_id] = track

            selected_tracks = []
            for track_id in curated_track_ids:
                if track_id in tracks_by_id:
                    track = tracks_by_id[track_id]

                    # Parse track_data_json if it's a string
                    track_data = track.track_data_json
                    if isinstance(track_data, str):
                        try:
                            track_data = json.loads(track_data)
                        except:
                            track_data = None

                    selected_tracks.append({
                        "track_id": track.spotify_track_id or track.itunes_track_id,
                        "spotify_track_id": track.spotify_track_id,
                        "itunes_track_id": track.itunes_track_id,
                        "track_name": track.track_name,
                        "artist_name": track.artist_name,
                        "album_name": track.album_name,
                        "album_cover_url": track.album_cover_url,
                        "duration_ms": track.duration_ms,
                        "track_data_json": track_data,
                        "source": track.source
                    })

            return jsonify({"success": True, "tracks": selected_tracks, "source": active_source})

        # Fallback: no curated playlist exists (shouldn't happen after first scan)
        return jsonify({"success": True, "tracks": [], "source": active_source})

    except Exception as e:
        print(f"Error getting discovery weekly: {e}")
        return jsonify({"success": False, "error": str(e)}), 500


@app.route('/api/discover/refresh', methods=['POST'])
def refresh_discover_data():
    """
    Force refresh discover page data (recent albums cache and curated playlists).
    Useful for initial setup or when data appears stale.
    """
    try:
        from core.watchlist_scanner import WatchlistScanner

        database = get_database()
        scanner = WatchlistScanner(spotify_client, database)

        print("[Discover Refresh] Starting forced refresh of discover data...")

        refresh_pid = get_current_profile_id()

        # Cache recent albums from watchlist and similar artists
        print("[Discover Refresh] Caching recent albums...")
        scanner.cache_discovery_recent_albums(profile_id=refresh_pid)

        # Curate playlists
        print("[Discover Refresh] Curating discovery playlists...")
        scanner.curate_discovery_playlists(profile_id=refresh_pid)

        # Get counts for response
        active_source = _get_active_discovery_source()
        pid = get_current_profile_id()
        recent_albums = database.get_discovery_recent_albums(limit=100, source=active_source, profile_id=pid)
        release_radar = database.get_curated_playlist(f'release_radar_{active_source}', profile_id=pid) or []
        discovery_weekly = database.get_curated_playlist(f'discovery_weekly_{active_source}', profile_id=pid) or []

        print(f"[Discover Refresh] Complete! Recent albums: {len(recent_albums)}, Release Radar: {len(release_radar)} tracks, Discovery Weekly: {len(discovery_weekly)} tracks")

        return jsonify({
            "success": True,
            "message": "Discover data refreshed",
            "source": active_source,
            "recent_albums_count": len(recent_albums),
            "release_radar_tracks": len(release_radar),
            "discovery_weekly_tracks": len(discovery_weekly)
        })

    except Exception as e:
        print(f"Error refreshing discover data: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500


@app.route('/api/discover/diagnose', methods=['GET'])
def diagnose_discover_data():
    """
    Diagnostic endpoint to check the state of discover data.
    Returns counts of similar artists, discovery pool, recent albums, etc.
    """
    try:
        database = get_database()
        active_source = _get_active_discovery_source()
        pid = get_current_profile_id()

        with database._get_connection() as conn:
            cursor = conn.cursor()

            # Similar artists stats
            cursor.execute("SELECT COUNT(*) as total FROM similar_artists WHERE profile_id = ?", (pid,))
            total_similar = cursor.fetchone()['total']

            cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_itunes_id IS NOT NULL AND profile_id = ?", (pid,))
            similar_with_itunes = cursor.fetchone()['count']

            cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_spotify_id IS NOT NULL AND profile_id = ?", (pid,))
            similar_with_spotify = cursor.fetchone()['count']

            # Discovery pool stats
            cursor.execute("SELECT source, COUNT(*) as count FROM discovery_pool WHERE profile_id = ? GROUP BY source", (pid,))
            pool_by_source = {row['source']: row['count'] for row in cursor.fetchall()}

            # Recent albums stats
            cursor.execute("SELECT source, COUNT(*) as count FROM discovery_recent_albums WHERE profile_id = ? GROUP BY source", (pid,))
            albums_by_source = {row['source']: row['count'] for row in cursor.fetchall()}

            # Curated playlists
            cursor.execute("SELECT playlist_type, track_ids_json FROM discovery_curated_playlists WHERE profile_id = ?", (pid,))
            playlists = {}
            for row in cursor.fetchall():
                import json
                track_ids = json.loads(row['track_ids_json']) if row['track_ids_json'] else []
                playlists[row['playlist_type']] = len(track_ids)

            # Watchlist artists
            cursor.execute("SELECT COUNT(*) as total FROM watchlist_artists WHERE profile_id = ?", (pid,))
            total_watchlist = cursor.fetchone()['total']

            cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE itunes_artist_id IS NOT NULL AND profile_id = ?", (pid,))
            watchlist_with_itunes = cursor.fetchone()['count']

        return jsonify({
            "success": True,
            "active_source": active_source,
            "similar_artists": {
                "total": total_similar,
                "with_itunes_id": similar_with_itunes,
                "with_spotify_id": similar_with_spotify
            },
            "discovery_pool": pool_by_source,
            "recent_albums": albums_by_source,
            "curated_playlists": playlists,
            "watchlist_artists": {
                "total": total_watchlist,
                "with_itunes_id": watchlist_with_itunes
            }
        })

    except Exception as e:
        print(f"Error diagnosing discover data: {e}")
        return jsonify({"success": False, "error": str(e)}), 500


# ========================================
# SEASONAL DISCOVERY ENDPOINTS
# ========================================

@app.route('/api/discover/seasonal/current', methods=['GET'])
def get_current_seasonal_content():
    """Auto-detect and return current season's content"""
    try:
        from core.seasonal_discovery import get_seasonal_discovery_service

        database = get_database()
        seasonal_service = get_seasonal_discovery_service(spotify_client, database)

        # Get current season
        current_season = seasonal_service.get_current_season()

        if not current_season:
            return jsonify({"success": True, "season": None, "albums": [], "playlist_available": False})

        # Get seasonal config
        from core.seasonal_discovery import SEASONAL_CONFIG
        config = SEASONAL_CONFIG[current_season]

        # Get albums for active source (increased limit for more variety)
        active_source = _get_active_discovery_source()
        albums = seasonal_service.get_seasonal_albums(current_season, limit=40, source=active_source)

        # Check if playlist is curated for active source
        playlist_track_ids = seasonal_service.get_curated_seasonal_playlist(current_season, source=active_source)

        return jsonify({
            "success": True,
            "season": current_season,
            "name": config['name'],
            "description": config['description'],
            "icon": config['icon'],
            "albums": albums,
            "playlist_available": len(playlist_track_ids) > 0
        })

    except Exception as e:
        print(f"Error getting current seasonal content: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/seasonal/<season_key>/albums', methods=['GET'])
def get_seasonal_albums(season_key):
    """Get albums for a specific season"""
    try:
        from core.seasonal_discovery import get_seasonal_discovery_service, SEASONAL_CONFIG

        if season_key not in SEASONAL_CONFIG:
            return jsonify({"success": False, "error": "Invalid season"}), 400

        database = get_database()
        seasonal_service = get_seasonal_discovery_service(spotify_client, database)

        active_source = _get_active_discovery_source()
        albums = seasonal_service.get_seasonal_albums(season_key, limit=40, source=active_source)
        config = SEASONAL_CONFIG[season_key]

        return jsonify({
            "success": True,
            "season": season_key,
            "name": config['name'],
            "description": config['description'],
            "icon": config['icon'],
            "albums": albums
        })

    except Exception as e:
        print(f"Error getting seasonal albums: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/seasonal/<season_key>/playlist', methods=['GET'])
def get_seasonal_playlist(season_key):
    """Get curated playlist for a specific season"""
    try:
        from core.seasonal_discovery import get_seasonal_discovery_service, SEASONAL_CONFIG

        if season_key not in SEASONAL_CONFIG:
            return jsonify({"success": False, "error": "Invalid season"}), 400

        database = get_database()
        seasonal_service = get_seasonal_discovery_service(spotify_client, database)

        # Get curated track IDs for active source
        active_source = _get_active_discovery_source()
        track_ids = seasonal_service.get_curated_seasonal_playlist(season_key, source=active_source)

        if not track_ids:
            return jsonify({"success": True, "tracks": []})

        # Use source-appropriate ID column for lookups
        track_id_col = 'spotify_track_id' if active_source == 'spotify' else 'itunes_track_id'

        # Fetch track details from seasonal tracks or discovery pool (filtered by source)
        tracks = []
        with database._get_connection() as conn:
            cursor = conn.cursor()

            for track_id in track_ids:
                # Try seasonal_tracks first (filtered by source)
                cursor.execute("""
                    SELECT
                        spotify_track_id,
                        track_name,
                        artist_name,
                        album_name,
                        album_cover_url,
                        duration_ms,
                        popularity,
                        track_data_json
                    FROM seasonal_tracks
                    WHERE spotify_track_id = ? AND source = ?
                """, (track_id, active_source))

                result = cursor.fetchone()

                if result:
                    track_dict = dict(result)
                    # Parse track_data_json if available
                    if track_dict.get('track_data_json'):
                        try:
                            import json
                            track_dict['track_data_json'] = json.loads(track_dict['track_data_json'])
                        except:
                            pass
                    tracks.append(track_dict)
                else:
                    # Try discovery_pool as fallback (filtered by source)
                    cursor.execute(f"""
                        SELECT
                            {track_id_col} as spotify_track_id,
                            track_name,
                            artist_name,
                            album_name,
                            album_cover_url,
                            duration_ms,
                            popularity,
                            track_data_json
                        FROM discovery_pool
                        WHERE {track_id_col} = ? AND source = ?
                    """, (track_id, active_source))

                    result = cursor.fetchone()
                    if result:
                        track_dict = dict(result)
                        # Parse track_data_json if available
                        if track_dict.get('track_data_json'):
                            try:
                                import json
                                track_dict['track_data_json'] = json.loads(track_dict['track_data_json'])
                            except:
                                pass
                        tracks.append(track_dict)

        config = SEASONAL_CONFIG[season_key]

        return jsonify({
            "success": True,
            "season": season_key,
            "name": config['name'],
            "description": config['description'],
            "icon": config['icon'],
            "tracks": tracks
        })

    except Exception as e:
        print(f"Error getting seasonal playlist: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/seasonal/refresh', methods=['POST'])
def refresh_seasonal_content():
    """Manually trigger seasonal content refresh (admin function)"""
    try:
        from core.seasonal_discovery import get_seasonal_discovery_service

        database = get_database()
        seasonal_service = get_seasonal_discovery_service(spotify_client, database)

        # Force populate current season in background thread (bypass 7-day threshold)
        import threading
        def populate_all():
            try:
                current_season = seasonal_service.get_current_season()
                if current_season:
                    print(f"🔄 Force-refreshing seasonal content for: {current_season}")
                    seasonal_service.populate_seasonal_content(current_season)
                    seasonal_service.curate_seasonal_playlist(current_season)
                    print(f"✅ Seasonal content refreshed for: {current_season}")
                else:
                    print("ℹ️ No active season to refresh")
            except Exception as e:
                print(f"Error in background seasonal population: {e}")

        thread = threading.Thread(target=populate_all, daemon=True)
        thread.start()

        return jsonify({"success": True, "message": "Seasonal content refresh started"})

    except Exception as e:
        print(f"Error refreshing seasonal content: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

# ========================================
# PERSONALIZED PLAYLISTS ENDPOINTS
# ========================================

@app.route('/api/discover/personalized/recently-added', methods=['GET'])
def get_recently_added_playlist():
    """Get recently added tracks from library"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        tracks = service.get_recently_added(limit=50)

        return jsonify({
            "success": True,
            "tracks": tracks
        })

    except Exception as e:
        print(f"Error getting recently added playlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/personalized/top-tracks', methods=['GET'])
def get_top_tracks_playlist():
    """Get user's all-time top tracks"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        tracks = service.get_top_tracks(limit=50)

        return jsonify({
            "success": True,
            "tracks": tracks
        })

    except Exception as e:
        print(f"Error getting top tracks playlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/personalized/forgotten-favorites', methods=['GET'])
def get_forgotten_favorites_playlist():
    """Get forgotten favorites - tracks you loved but haven't played recently"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        tracks = service.get_forgotten_favorites(limit=50)

        return jsonify({
            "success": True,
            "tracks": tracks
        })

    except Exception as e:
        print(f"Error getting forgotten favorites playlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/personalized/decade/<int:decade>', methods=['GET'])
def get_decade_playlist(decade):
    """Get tracks from a specific decade"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        tracks = service.get_decade_playlist(decade, limit=100)

        return jsonify({
            "success": True,
            "decade": decade,
            "tracks": tracks
        })

    except Exception as e:
        print(f"Error getting decade playlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/personalized/popular-picks', methods=['GET'])
def get_popular_picks_playlist():
    """Get high popularity tracks from discovery pool"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        tracks = service.get_popular_picks(limit=50)

        return jsonify({
            "success": True,
            "tracks": tracks
        })

    except Exception as e:
        print(f"Error getting popular picks playlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/personalized/hidden-gems', methods=['GET'])
def get_hidden_gems_playlist():
    """Get hidden gems (low popularity) from discovery pool"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        tracks = service.get_hidden_gems(limit=50)

        return jsonify({
            "success": True,
            "tracks": tracks
        })

    except Exception as e:
        print(f"Error getting hidden gems playlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/personalized/daily-mixes', methods=['GET'])
def get_daily_mixes():
    """Get all Daily Mix playlists (hybrid library + discovery)"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        mixes = service.get_all_daily_mixes(max_mixes=4)

        return jsonify({
            "success": True,
            "mixes": mixes
        })

    except Exception as e:
        print(f"Error getting daily mixes: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/personalized/discovery-shuffle', methods=['GET'])
def get_discovery_shuffle():
    """Get Discovery Shuffle playlist - random tracks from discovery pool"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        limit = int(request.args.get('limit', 50))
        tracks = service.get_discovery_shuffle(limit=limit)

        return jsonify({
            "success": True,
            "tracks": tracks
        })

    except Exception as e:
        print(f"Error getting discovery shuffle playlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/personalized/familiar-favorites', methods=['GET'])
def get_familiar_favorites():
    """Get Familiar Favorites playlist - reliable go-to tracks"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        limit = int(request.args.get('limit', 50))
        tracks = service.get_familiar_favorites(limit=limit)

        return jsonify({
            "success": True,
            "tracks": tracks
        })

    except Exception as e:
        print(f"Error getting familiar favorites playlist: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/build-playlist/search-artists', methods=['GET'])
def search_artists_for_playlist():
    """Search for artists to use as seeds for custom playlist building"""
    try:
        query = request.args.get('query', '').strip()
        if not query:
            return jsonify({"success": False, "error": "Query required"}), 400

        artists = []
        if _is_hydrabase_active():
            artist_objs = hydrabase_client.search_artists(query, limit=10)
            for artist in artist_objs:
                artists.append({
                    'id': artist.id,
                    'name': artist.name,
                    'image_url': artist.image_url
                })
        else:
            if hydrabase_worker and dev_mode_enabled:
                hydrabase_worker.enqueue(query, 'artists')

            search_query = f'artist:{query}' if len(query.strip()) <= 4 else query
            results = spotify_client.sp.search(q=search_query, type='artist', limit=10)
            if results and 'artists' in results and 'items' in results['artists']:
                for artist in results['artists']['items']:
                    artists.append({
                        'id': artist['id'],
                        'name': artist['name'],
                        'image_url': artist['images'][0]['url'] if artist.get('images') else None
                    })
                # Re-rank: boost exact name matches to the top
                query_lower = query.lower().strip()
                artists.sort(key=lambda a: (0 if a['name'].lower().strip() == query_lower else 1))

        return jsonify({
            "success": True,
            "artists": artists
        })

    except Exception as e:
        print(f"Error searching for artists: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/build-playlist/generate', methods=['POST'])
def generate_custom_playlist():
    """Generate custom playlist from seed artists"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        data = request.get_json()
        seed_artist_ids = data.get('seed_artist_ids', [])

        if not seed_artist_ids or len(seed_artist_ids) < 1 or len(seed_artist_ids) > 5:
            return jsonify({
                "success": False,
                "error": "Please provide between 1 and 5 seed artists"
            }), 400

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        playlist_size = int(data.get('playlist_size', 50))
        result = service.build_custom_playlist(seed_artist_ids, playlist_size=playlist_size)

        return jsonify({
            "success": True,
            "playlist": result
        })

    except Exception as e:
        print(f"Error generating custom playlist: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/decades/available', methods=['GET'])
def get_available_decades():
    """Get list of decades that have content in discovery pool"""
    try:
        database = get_database()

        with database._get_connection() as conn:
            cursor = conn.cursor()

            # Get distinct decades from discovery pool
            cursor.execute("""
                SELECT DISTINCT
                    (CAST(SUBSTR(release_date, 1, 4) AS INTEGER) / 10) * 10 as decade,
                    COUNT(*) as track_count
                FROM discovery_pool
                WHERE release_date IS NOT NULL
                  AND CAST(SUBSTR(release_date, 1, 4) AS INTEGER) >= 1950
                  AND CAST(SUBSTR(release_date, 1, 4) AS INTEGER) <= 2029
                GROUP BY decade
                HAVING track_count >= 10
                ORDER BY decade ASC
            """)

            rows = cursor.fetchall()
            decades = []
            for row in rows:
                decades.append({
                    'year': row[0],
                    'track_count': row[1]
                })

            return jsonify({
                "success": True,
                "decades": decades
            })

    except Exception as e:
        print(f"Error getting available decades: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/decade/<int:decade>', methods=['GET'])
def get_discover_decade_playlist(decade):
    """Get tracks from a specific decade for discovery page"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        tracks = service.get_decade_playlist(decade, limit=50)

        if not tracks:
            return jsonify({
                "success": True,
                "tracks": [],
                "decade": decade,
                "message": f"No tracks found for the {decade}s"
            }), 200

        # Convert to Spotify format for modal compatibility
        spotify_tracks = []
        for track in tracks:
            spotify_tracks.append({
                'id': track.get('spotify_track_id', track.get('id')),
                'name': track.get('track_name', track.get('name')),
                'artists': [track.get('artist_name', 'Unknown')],
                'album': {
                    'name': track.get('album_name', 'Unknown'),
                    'images': [{'url': track.get('album_cover_url')}] if track.get('album_cover_url') else []
                },
                'duration_ms': track.get('duration_ms', 0)
            })

        return jsonify({
            "success": True,
            "tracks": spotify_tracks,
            "decade": decade
        })

    except Exception as e:
        print(f"Error getting decade playlist: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/genres/available', methods=['GET'])
def get_available_genres():
    """Get list of genres that have content in discovery pool"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        genres = service.get_available_genres()

        return jsonify({
            "success": True,
            "genres": genres
        })

    except Exception as e:
        print(f"Error getting available genres: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/genre/<path:genre_name>', methods=['GET'])
def get_discover_genre_playlist(genre_name):
    """Get tracks from a specific genre for discovery page"""
    try:
        from core.personalized_playlists import get_personalized_playlists_service

        database = get_database()
        service = get_personalized_playlists_service(database, spotify_client)

        tracks = service.get_genre_playlist(genre_name, limit=50)

        if not tracks:
            return jsonify({
                "success": True,
                "tracks": [],
                "genre": genre_name,
                "message": f"No tracks found for {genre_name}"
            }), 200

        # Convert to Spotify format for modal compatibility
        spotify_tracks = []
        for track in tracks:
            spotify_tracks.append({
                'id': track.get('spotify_track_id', track.get('id')),
                'name': track.get('track_name', track.get('name')),
                'artists': [track.get('artist_name', 'Unknown')],
                'album': {
                    'name': track.get('album_name', 'Unknown'),
                    'images': [{'url': track.get('album_cover_url')}] if track.get('album_cover_url') else []
                },
                'duration_ms': track.get('duration_ms', 0)
            })

        return jsonify({
            "success": True,
            "tracks": spotify_tracks,
            "genre": genre_name
        })

    except Exception as e:
        print(f"Error getting genre playlist: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

# ===============================
# LISTENBRAINZ DISCOVER ENDPOINTS
# ===============================

@app.route('/api/discover/listenbrainz/created-for', methods=['GET'])
def get_listenbrainz_created_for():
    """Get playlists created for the user by ListenBrainz (from cache)"""
    try:
        from core.listenbrainz_manager import ListenBrainzManager

        lb_manager = ListenBrainzManager(str(get_database().database_path))

        # Check if cache is empty - if so, populate it on first load
        if not lb_manager.has_cached_playlists():
            # Check if authenticated
            if not lb_manager.client.is_authenticated():
                return jsonify({
                    "success": False,
                    "error": "Not authenticated",
                    "playlists": [],
                    "count": 0
                })

            # Populate cache on first load
            print("📦 Cache empty, populating ListenBrainz playlists...")
            lb_manager.update_all_playlists()

        playlists = lb_manager.get_cached_playlists('created_for')

        # Convert to JSPF-like format for frontend compatibility
        formatted_playlists = []
        for playlist in playlists:
            formatted_playlists.append({
                "playlist": {
                    "identifier": f"https://listenbrainz.org/playlist/{playlist['playlist_mbid']}",
                    "title": playlist['title'],
                    "creator": playlist['creator'],
                    "annotation": playlist.get('annotation', {}),
                    "track": []  # Track count is in annotation
                }
            })

        return jsonify({
            "success": True,
            "playlists": formatted_playlists,
            "count": len(formatted_playlists)
        })

    except Exception as e:
        print(f"Error getting cached ListenBrainz created-for playlists: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/listenbrainz/user-playlists', methods=['GET'])
def get_listenbrainz_user_playlists():
    """Get user's own ListenBrainz playlists (from cache)"""
    try:
        from core.listenbrainz_manager import ListenBrainzManager

        lb_manager = ListenBrainzManager(str(get_database().database_path))

        # Check if cache is empty - if so, populate it on first load
        if not lb_manager.has_cached_playlists():
            # Check if authenticated
            if not lb_manager.client.is_authenticated():
                return jsonify({
                    "success": False,
                    "error": "Not authenticated",
                    "playlists": [],
                    "count": 0
                })

            # Populate cache on first load
            print("📦 Cache empty, populating ListenBrainz playlists...")
            lb_manager.update_all_playlists()

        playlists = lb_manager.get_cached_playlists('user')

        # Convert to JSPF-like format for frontend compatibility
        formatted_playlists = []
        for playlist in playlists:
            formatted_playlists.append({
                "playlist": {
                    "identifier": f"https://listenbrainz.org/playlist/{playlist['playlist_mbid']}",
                    "title": playlist['title'],
                    "creator": playlist['creator'],
                    "annotation": playlist.get('annotation', {}),
                    "track": []
                }
            })

        return jsonify({
            "success": True,
            "playlists": formatted_playlists,
            "count": len(formatted_playlists)
        })

    except Exception as e:
        print(f"Error getting cached ListenBrainz user playlists: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/listenbrainz/collaborative', methods=['GET'])
def get_listenbrainz_collaborative():
    """Get collaborative ListenBrainz playlists (from cache)"""
    try:
        from core.listenbrainz_manager import ListenBrainzManager

        lb_manager = ListenBrainzManager(str(get_database().database_path))

        # Check if cache is empty - if so, populate it on first load
        if not lb_manager.has_cached_playlists():
            # Check if authenticated
            if not lb_manager.client.is_authenticated():
                return jsonify({
                    "success": False,
                    "error": "Not authenticated",
                    "playlists": [],
                    "count": 0
                })

            # Populate cache on first load
            print("📦 Cache empty, populating ListenBrainz playlists...")
            lb_manager.update_all_playlists()

        playlists = lb_manager.get_cached_playlists('collaborative')

        # Convert to JSPF-like format for frontend compatibility
        formatted_playlists = []
        for playlist in playlists:
            formatted_playlists.append({
                "playlist": {
                    "identifier": f"https://listenbrainz.org/playlist/{playlist['playlist_mbid']}",
                    "title": playlist['title'],
                    "creator": playlist['creator'],
                    "annotation": playlist.get('annotation', {}),
                    "track": []
                }
            })

        return jsonify({
            "success": True,
            "playlists": formatted_playlists,
            "count": len(formatted_playlists)
        })

    except Exception as e:
        print(f"Error getting cached ListenBrainz collaborative playlists: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/discover/listenbrainz/playlist/<playlist_mbid>', methods=['GET'])
def get_listenbrainz_playlist_tracks(playlist_mbid):
    """Get tracks from a specific ListenBrainz playlist (from cache)"""
    try:
        from core.listenbrainz_manager import ListenBrainzManager

        lb_manager = ListenBrainzManager(str(get_database().database_path))
        tracks = lb_manager.get_cached_tracks(playlist_mbid)

        if not tracks:
            return jsonify({
                "success": False,
                "error": "Playlist not found in cache"
            }), 404

        return jsonify({
            "success": True,
            "tracks": tracks,
            "track_count": len(tracks)
        })

    except Exception as e:
        print(f"Error getting cached ListenBrainz playlist tracks: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500


# Manual refresh endpoint for ListenBrainz
@app.route('/api/discover/listenbrainz/refresh', methods=['POST'])
def refresh_listenbrainz():
    """Manually refresh ListenBrainz playlists cache"""
    try:
        from core.listenbrainz_manager import ListenBrainzManager

        lb_manager = ListenBrainzManager(str(get_database().database_path))
        result = lb_manager.update_all_playlists()

        return jsonify(result)

    except Exception as e:
        print(f"Error refreshing ListenBrainz: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

# ========================================
# LISTENBRAINZ PLAYLIST MANAGEMENT (Discovery System)
# ========================================

@app.route('/api/listenbrainz/playlists', methods=['GET'])
def get_all_listenbrainz_playlists():
    """Get all stored ListenBrainz playlists for frontend hydration"""
    try:
        playlists = []
        current_time = time.time()

        for playlist_mbid, state in listenbrainz_playlist_states.items():
            # Update access time when requested
            state['last_accessed'] = current_time

            # Return essential data for card recreation
            playlist_info = {
                'playlist_mbid': playlist_mbid,
                'playlist': state['playlist'],
                'phase': state['phase'],
                'status': state['status'],
                'discovery_progress': state['discovery_progress'],
                'spotify_matches': state['spotify_matches'],
                'spotify_total': state['spotify_total'],
                'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'),
                'download_process_id': state.get('download_process_id'),
                'created_at': state['created_at'],
                'last_accessed': state['last_accessed']
            }
            playlists.append(playlist_info)

        print(f"📋 Returning {len(playlists)} stored ListenBrainz playlists for hydration")
        return jsonify({"playlists": playlists})

    except Exception as e:
        print(f"❌ Error getting ListenBrainz playlists: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/listenbrainz/state/<playlist_mbid>', methods=['GET'])
def get_listenbrainz_playlist_state(playlist_mbid):
    """Get specific ListenBrainz playlist state (detailed version)"""
    try:
        if playlist_mbid not in listenbrainz_playlist_states:
            return jsonify({"error": "ListenBrainz playlist not found"}), 404

        state = listenbrainz_playlist_states[playlist_mbid]
        state['last_accessed'] = time.time()

        # Return full state information (including results for modal hydration)
        response = {
            'playlist_mbid': playlist_mbid,
            'playlist': state['playlist'],
            'phase': state['phase'],
            'status': state['status'],
            'discovery_progress': state['discovery_progress'],
            'spotify_matches': state['spotify_matches'],
            'spotify_total': state['spotify_total'],
            'discovery_results': state['discovery_results'],
            'sync_playlist_id': state.get('sync_playlist_id'),
            'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'),
            'download_process_id': state.get('download_process_id'),
            'sync_progress': state.get('sync_progress', {}),
            'created_at': state['created_at'],
            'last_accessed': state['last_accessed']
        }

        return jsonify(response)

    except Exception as e:
        print(f"❌ Error getting ListenBrainz playlist state: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/listenbrainz/reset/<playlist_mbid>', methods=['POST'])
def reset_listenbrainz_playlist(playlist_mbid):
    """Reset ListenBrainz playlist to fresh phase (clear discovery/sync data)"""
    try:
        if playlist_mbid not in listenbrainz_playlist_states:
            return jsonify({"error": "ListenBrainz playlist not found"}), 404

        state = listenbrainz_playlist_states[playlist_mbid]

        # Stop any active discovery
        if 'discovery_future' in state and state['discovery_future']:
            state['discovery_future'].cancel()

        # Reset state to fresh (preserve original playlist data)
        state['phase'] = 'fresh'
        state['status'] = 'cached'
        state['discovery_results'] = []
        state['discovery_progress'] = 0
        state['spotify_matches'] = 0
        state['sync_playlist_id'] = None
        state['converted_spotify_playlist_id'] = None
        state['sync_progress'] = {}
        state['discovery_future'] = None
        state['last_accessed'] = time.time()

        print(f"🔄 Reset ListenBrainz playlist to fresh: {state['playlist']['title']}")
        return jsonify({"success": True, "phase": "fresh"})

    except Exception as e:
        print(f"❌ Error resetting ListenBrainz playlist: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/listenbrainz/remove/<playlist_mbid>', methods=['POST'])
def remove_listenbrainz_playlist(playlist_mbid):
    """Remove ListenBrainz playlist from state (doesn't affect cache)"""
    try:
        if playlist_mbid not in listenbrainz_playlist_states:
            return jsonify({"error": "ListenBrainz playlist not found"}), 404

        state = listenbrainz_playlist_states[playlist_mbid]

        # Stop any active discovery
        if 'discovery_future' in state and state['discovery_future']:
            state['discovery_future'].cancel()

        # Remove from state
        del listenbrainz_playlist_states[playlist_mbid]

        print(f"🗑️ Removed ListenBrainz playlist from state: {playlist_mbid}")
        return jsonify({"success": True})

    except Exception as e:
        print(f"❌ Error removing ListenBrainz playlist: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/listenbrainz/discovery/start/<playlist_mbid>', methods=['POST'])
def start_listenbrainz_discovery(playlist_mbid):
    """Initialize and start Spotify discovery process for a ListenBrainz playlist"""
    try:
        data = request.get_json()
        playlist_data = data.get('playlist')

        if not playlist_data:
            return jsonify({"error": "Playlist data required"}), 400

        # Create or update state
        if playlist_mbid not in listenbrainz_playlist_states:
            # Initialize new state
            listenbrainz_playlist_states[playlist_mbid] = {
                'playlist_mbid': playlist_mbid,
                'playlist': playlist_data,
                'phase': 'discovering',
                'status': 'discovering',
                'discovery_progress': 0,
                'spotify_matches': 0,
                'spotify_total': len(playlist_data.get('tracks', [])),
                'discovery_results': [],
                'created_at': time.time(),
                'last_accessed': time.time()
            }
            print(f"✅ Created new ListenBrainz playlist state: {playlist_data.get('name', 'Unknown')}")
        else:
            # State already exists, update it
            state = listenbrainz_playlist_states[playlist_mbid]
            if state['phase'] == 'discovering':
                return jsonify({"error": "Discovery already in progress"}), 400

            # Reset for new discovery
            state['phase'] = 'discovering'
            state['status'] = 'discovering'
            state['discovery_progress'] = 0
            state['spotify_matches'] = 0
            state['discovery_results'] = []
            state['last_accessed'] = time.time()

        state = listenbrainz_playlist_states[playlist_mbid]

        # Add activity for discovery start
        playlist_name = playlist_data.get('name', 'Unknown Playlist')
        track_count = len(playlist_data.get('tracks', []))
        add_activity_item("🔍", "ListenBrainz Discovery Started", f"'{playlist_name}' - {track_count} tracks", "Now")

        # Start discovery worker
        future = listenbrainz_discovery_executor.submit(_run_listenbrainz_discovery_worker, playlist_mbid)
        state['discovery_future'] = future

        print(f"🔍 Started Spotify discovery for ListenBrainz playlist: {playlist_name}")
        return jsonify({"success": True, "message": "Discovery started"})

    except Exception as e:
        print(f"❌ Error starting ListenBrainz discovery: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"error": str(e)}), 500

@app.route('/api/listenbrainz/discovery/status/<playlist_mbid>', methods=['GET'])
def get_listenbrainz_discovery_status(playlist_mbid):
    """Get real-time discovery status for a ListenBrainz playlist"""
    try:
        if playlist_mbid not in listenbrainz_playlist_states:
            return jsonify({"error": "ListenBrainz playlist not found"}), 404

        state = listenbrainz_playlist_states[playlist_mbid]
        state['last_accessed'] = time.time()

        response = {
            'phase': state['phase'],
            'status': state['status'],
            'progress': state['discovery_progress'],
            'spotify_matches': state['spotify_matches'],
            'spotify_total': state['spotify_total'],
            'results': state['discovery_results'],
            'complete': state['phase'] == 'discovered'
        }

        return jsonify(response)

    except Exception as e:
        print(f"❌ Error getting ListenBrainz discovery status: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/listenbrainz/update-phase/<playlist_mbid>', methods=['POST'])
def update_listenbrainz_phase(playlist_mbid):
    """Update ListenBrainz playlist phase (for phase transitions and persistence)"""
    try:
        if playlist_mbid not in listenbrainz_playlist_states:
            return jsonify({"error": "ListenBrainz playlist not found"}), 404

        data = request.get_json() or {}
        new_phase = data.get('phase')

        if not new_phase:
            return jsonify({"error": "Phase is required"}), 400

        state = listenbrainz_playlist_states[playlist_mbid]
        state['phase'] = new_phase
        state['last_accessed'] = time.time()

        # Update download process ID if provided (for download persistence)
        if 'download_process_id' in data:
            state['download_process_id'] = data['download_process_id']
            logger.info(f"🎵 Updated ListenBrainz download_process_id: {data['download_process_id']}")

        # Update converted Spotify playlist ID if provided (for download persistence)
        if 'converted_spotify_playlist_id' in data:
            state['converted_spotify_playlist_id'] = data['converted_spotify_playlist_id']
            logger.info(f"🎵 Updated ListenBrainz converted_spotify_playlist_id: {data['converted_spotify_playlist_id']}")

        logger.info(f"🎵 Updated ListenBrainz playlist {playlist_mbid} phase to: {new_phase}")

        return jsonify({
            "success": True,
            "phase": new_phase
        })

    except Exception as e:
        print(f"❌ Error updating ListenBrainz playlist phase: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/listenbrainz/discovery/update_match', methods=['POST'])
def update_listenbrainz_discovery_match():
    """Update a ListenBrainz discovery result with manually selected Spotify track"""
    try:
        data = request.get_json()
        identifier = data.get('identifier')  # playlist_mbid
        track_index = data.get('track_index')
        spotify_track = data.get('spotify_track')

        if not identifier or track_index is None or not spotify_track:
            return jsonify({'error': 'Missing required fields'}), 400

        # Get the state
        state = listenbrainz_playlist_states.get(identifier)

        if not state:
            return jsonify({'error': 'Discovery state not found'}), 404

        # Update the discovery result
        if track_index < len(state['discovery_results']):
            result = state['discovery_results'][track_index]

            # Was previously not found, now found
            if result['status_class'] == 'not-found' and spotify_track:
                state['spotify_matches'] += 1
            # Was previously found, now not found
            elif result['status_class'] == 'found' and not spotify_track:
                state['spotify_matches'] -= 1

            # Update result
            result['status'] = '✅ Found' if spotify_track else '❌ Not Found'
            result['status_class'] = 'found' if spotify_track else 'not-found'
            result['spotify_track'] = spotify_track.get('name', '') if spotify_track else ''
            # Join all artists (matching YouTube/Tidal/Beatport format)
            artists = spotify_track.get('artists', []) if spotify_track else []
            result['spotify_artist'] = ', '.join(artists) if isinstance(artists, list) else artists
            # Album comes as a string from the frontend fix modal
            album = spotify_track.get('album', '') if spotify_track else ''
            result['spotify_album'] = album if isinstance(album, str) else album.get('name', '') if isinstance(album, dict) else ''
            result['spotify_id'] = spotify_track.get('id', '') if spotify_track else ''

            if spotify_track:
                # Store spotify_data in the same format as other platforms
                result['spotify_data'] = {
                    'id': spotify_track.get('id', ''),
                    'name': spotify_track.get('name', ''),
                    'artists': artists if isinstance(artists, list) else [artists],
                    'album': result['spotify_album'],
                    'duration_ms': spotify_track.get('duration_ms', 0)
                }
            else:
                result['spotify_data'] = None

            result['manual_match'] = True

            print(f"✅ Updated ListenBrainz match for track {track_index}: {result['status']}")
            return jsonify({'success': True})
        else:
            return jsonify({'error': 'Invalid track index'}), 400

    except Exception as e:
        print(f"❌ Error updating ListenBrainz discovery match: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({'error': str(e)}), 500

def convert_listenbrainz_results_to_spotify_tracks(discovery_results):
    """Convert ListenBrainz discovery results to Spotify tracks format for sync"""
    spotify_tracks = []

    for result in discovery_results:
        # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery)
        if result.get('spotify_data'):
            spotify_data = result['spotify_data']

            # Create track object matching the expected format
            track = {
                'id': spotify_data['id'],
                'name': spotify_data['name'],
                'artists': spotify_data['artists'],
                'album': spotify_data['album'],
                'duration_ms': spotify_data.get('duration_ms', 0)
            }
            spotify_tracks.append(track)
        elif result.get('spotify_track') and result.get('status_class') == 'found':
            # Build from individual fields (automatic discovery format)
            track = {
                'id': result.get('spotify_id', 'unknown'),
                'name': result.get('spotify_track', 'Unknown Track'),
                'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'],
                'album': result.get('spotify_album', 'Unknown Album'),
                'duration_ms': 0
            }
            spotify_tracks.append(track)

    print(f"🔄 Converted {len(spotify_tracks)} ListenBrainz matches to Spotify tracks for sync")
    return spotify_tracks

@app.route('/api/listenbrainz/sync/start/<playlist_mbid>', methods=['POST'])
def start_listenbrainz_sync(playlist_mbid):
    """Start sync process for a ListenBrainz playlist using discovered Spotify tracks"""
    try:
        if playlist_mbid not in listenbrainz_playlist_states:
            return jsonify({"error": "ListenBrainz playlist not found"}), 404

        state = listenbrainz_playlist_states[playlist_mbid]
        state['last_accessed'] = time.time()  # Update access time

        if state['phase'] not in ['discovered', 'sync_complete']:
            return jsonify({"error": "ListenBrainz playlist not ready for sync"}), 400

        # Convert discovery results to Spotify tracks format
        spotify_tracks = convert_listenbrainz_results_to_spotify_tracks(state['discovery_results'])

        if not spotify_tracks:
            return jsonify({"error": "No Spotify matches found for sync"}), 400

        # Create a temporary playlist ID for sync tracking
        sync_playlist_id = f"listenbrainz_{playlist_mbid}"
        playlist_name = state['playlist']['name']

        # Add activity for sync start
        add_activity_item("🔄", "ListenBrainz Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now")

        # Update ListenBrainz state
        state['phase'] = 'syncing'
        state['sync_playlist_id'] = sync_playlist_id
        state['sync_progress'] = {}

        # Start the sync using existing sync infrastructure
        sync_data = {
            'playlist_id': sync_playlist_id,
            'playlist_name': playlist_name,
            'tracks': spotify_tracks
        }

        with sync_lock:
            sync_states[sync_playlist_id] = {"status": "starting", "progress": {}}

        # Submit sync task
        future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks)
        active_sync_workers[sync_playlist_id] = future

        print(f"🔄 Started ListenBrainz sync for: {playlist_name} ({len(spotify_tracks)} tracks)")
        return jsonify({"success": True, "sync_playlist_id": sync_playlist_id})

    except Exception as e:
        print(f"❌ Error starting ListenBrainz sync: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/listenbrainz/sync/status/<playlist_mbid>', methods=['GET'])
def get_listenbrainz_sync_status(playlist_mbid):
    """Get sync status for a ListenBrainz playlist"""
    try:
        if playlist_mbid not in listenbrainz_playlist_states:
            return jsonify({"error": "ListenBrainz playlist not found"}), 404

        state = listenbrainz_playlist_states[playlist_mbid]
        state['last_accessed'] = time.time()  # Update access time
        sync_playlist_id = state.get('sync_playlist_id')

        if not sync_playlist_id:
            return jsonify({"error": "No sync in progress"}), 404

        # Get sync status from existing sync infrastructure
        with sync_lock:
            sync_state = sync_states.get(sync_playlist_id, {})

        response = {
            'phase': state['phase'],
            'sync_status': sync_state.get('status', 'unknown'),
            'progress': sync_state.get('progress', {}),
            'complete': sync_state.get('status') == 'finished',
            'error': sync_state.get('error')
        }

        # Update ListenBrainz state if sync completed
        if sync_state.get('status') == 'finished':
            state['phase'] = 'sync_complete'
            state['sync_progress'] = sync_state.get('progress', {})
            # Add activity for sync completion
            playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist')
            add_activity_item("🔄", "Sync Complete", f"ListenBrainz playlist '{playlist_name}' synced successfully", "Now")
        elif sync_state.get('status') == 'error':
            state['phase'] = 'discovered'  # Revert on error
            playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist')
            add_activity_item("❌", "Sync Failed", f"ListenBrainz playlist '{playlist_name}' sync failed", "Now")

        return jsonify(response)

    except Exception as e:
        print(f"❌ Error getting ListenBrainz sync status: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/listenbrainz/sync/cancel/<playlist_mbid>', methods=['POST'])
def cancel_listenbrainz_sync(playlist_mbid):
    """Cancel sync for a ListenBrainz playlist"""
    try:
        if playlist_mbid not in listenbrainz_playlist_states:
            return jsonify({"error": "ListenBrainz playlist not found"}), 404

        state = listenbrainz_playlist_states[playlist_mbid]
        state['last_accessed'] = time.time()  # Update access time
        sync_playlist_id = state.get('sync_playlist_id')

        if sync_playlist_id:
            # Cancel the sync using existing sync infrastructure
            with sync_lock:
                sync_states[sync_playlist_id] = {"status": "cancelled"}

            # Clean up sync worker
            if sync_playlist_id in active_sync_workers:
                del active_sync_workers[sync_playlist_id]

        # Revert ListenBrainz state
        state['phase'] = 'discovered'
        state['sync_playlist_id'] = None
        state['sync_progress'] = {}

        return jsonify({"success": True, "message": "ListenBrainz sync cancelled"})

    except Exception as e:
        print(f"❌ Error cancelling ListenBrainz sync: {e}")
        return jsonify({"error": str(e)}), 500


# OLD ENDPOINT - REMOVE ALL THE CODE BELOW FOR THE OLD IMPLEMENTATION
def _old_get_listenbrainz_playlist_tracks_DEPRECATED(playlist_mbid):
    """DEPRECATED - Old implementation that fetches from API"""
    try:
        from core.listenbrainz_client import ListenBrainzClient

        client = ListenBrainzClient()

        playlist = client.get_playlist_details(playlist_mbid, fetch_metadata=True)

        if not playlist:
            return jsonify({
                "success": False,
                "error": "Playlist not found or not accessible"
            }), 404

        # Extract tracks from JSPF format
        jspf_tracks = playlist.get('track', [])

        # Convert to our standard format - prepare tracks first without cover art
        tracks = []
        print(f"🎵 Processing {len(jspf_tracks)} tracks from playlist")

        # First pass: extract all track data without cover art
        track_data_list = []
        for idx, track in enumerate(jspf_tracks):
            # Get recording MBID from identifier
            recording_mbid = None
            identifiers = track.get('identifier', [])
            for identifier in identifiers:
                if 'musicbrainz.org/recording/' in identifier:
                    recording_mbid = identifier.split('/')[-1]
                    break

            # Get extension data (has MusicBrainz metadata)
            extension = track.get('extension', {})
            mb_data = extension.get('https://musicbrainz.org/doc/jspf#track', {})

            if idx == 0:
                print(f"📋 Sample track extension data: {extension}")
                print(f"📋 Sample mb_data keys: {mb_data.keys() if mb_data else 'None'}")

            # Extract release MBID for cover art
            release_mbid = None
            if mb_data:
                # Check in additional_metadata first
                additional_metadata = mb_data.get('additional_metadata', {})
                if 'caa_release_mbid' in additional_metadata:
                    release_mbid = additional_metadata['caa_release_mbid']
                # Fallback to top-level release_mbid
                elif 'release_mbid' in mb_data:
                    release_mbid = mb_data['release_mbid']

            if idx == 0:
                print(f"🆔 First track release_mbid: {release_mbid}")

            track_data = {
                'track_name': track.get('title', 'Unknown Track'),
                'artist_name': track.get('creator', 'Unknown Artist'),
                'album_name': track.get('album', 'Unknown Album'),
                'duration_ms': track.get('duration', 0),
                'mbid': recording_mbid,
                'release_mbid': release_mbid,
                'album_cover_url': None,  # Will be fetched in parallel
                'additional_metadata': mb_data
            }

            track_data_list.append(track_data)

        # Second pass: fetch cover art in parallel using threading (much faster)
        from concurrent.futures import ThreadPoolExecutor, as_completed
        import time

        def fetch_cover_art(track_data):
            """Fetch cover art for a single track"""
            release_mbid = track_data.get('release_mbid')
            if not release_mbid:
                return None

            try:
                cover_art_url = f"https://coverartarchive.org/release/{release_mbid}"
                cover_response = requests.get(cover_art_url, timeout=3)

                if cover_response.status_code == 200:
                    cover_data = cover_response.json()
                    images = cover_data.get('images', [])

                    # Get front cover
                    for img in images:
                        if img.get('front'):
                            return img.get('thumbnails', {}).get('small') or img.get('image')

                    # Fallback to first image
                    if images:
                        return images[0].get('thumbnails', {}).get('small') or images[0].get('image')
            except:
                pass

            return None

        print(f"🎨 Fetching cover art for {len(track_data_list)} tracks in parallel...")
        start_time = time.time()

        # Fetch up to 10 covers at a time
        with ThreadPoolExecutor(max_workers=10) as executor:
            future_to_track = {executor.submit(fetch_cover_art, track): idx
                             for idx, track in enumerate(track_data_list)}

            for future in as_completed(future_to_track):
                idx = future_to_track[future]
                try:
                    cover_url = future.result()
                    if cover_url:
                        track_data_list[idx]['album_cover_url'] = cover_url
                except Exception as e:
                    pass

        elapsed = time.time() - start_time
        covers_found = sum(1 for t in track_data_list if t.get('album_cover_url'))
        print(f"✅ Fetched {covers_found}/{len(track_data_list)} covers in {elapsed:.2f}s")

        tracks = track_data_list

        return jsonify({
            "success": True,
            "tracks": tracks,
            "track_count": len(tracks)
        })

    except Exception as e:
        print(f"Error getting ListenBrainz playlist tracks: {e}")
        import traceback
        traceback.print_exc()
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/metadata/start', methods=['POST'])
def start_metadata_update():
    """Start the metadata update process - EXACT copy of dashboard.py logic"""
    global metadata_update_worker, metadata_update_state
    
    try:
        # Check if already running
        if metadata_update_state['status'] == 'running':
            return jsonify({"success": False, "error": "Metadata update already running"}), 400
        
        # Get refresh interval from request
        data = request.get_json() or {}
        refresh_interval_days = data.get('refresh_interval_days', 30)
        
        # Check active server and client availability - EXACTLY like dashboard.py
        active_server = config_manager.get_active_media_server()
        
        # Get appropriate media client - Support all three servers
        if active_server == "jellyfin":
            media_client = jellyfin_client
            if not media_client:
                add_activity_item("❌", "Metadata Update", "Jellyfin client not available", "Now")
                return jsonify({"success": False, "error": "Jellyfin client not available"}), 400
        elif active_server == "navidrome":
            media_client = navidrome_client
            if not media_client:
                add_activity_item("❌", "Metadata Update", "Navidrome client not available", "Now")
                return jsonify({"success": False, "error": "Navidrome client not available"}), 400
        else:  # plex
            media_client = plex_client
            if not media_client:
                add_activity_item("❌", "Metadata Update", "Plex client not available", "Now")
                return jsonify({"success": False, "error": "Plex client not available"}), 400
            
            # DEBUG: Check Plex connection details
            print(f"[DEBUG] Active server: {active_server}")
            print(f"[DEBUG] Plex client: {media_client}")
            if hasattr(media_client, 'server') and media_client.server:
                print(f"[DEBUG] Plex server URL: {getattr(media_client.server, '_baseurl', 'NO_URL')}")
                print(f"[DEBUG] Plex server name: {getattr(media_client.server, 'friendlyName', 'NO_NAME')}")
                # Check available libraries
                try:
                    sections = media_client.server.library.sections()
                    print(f"[DEBUG] Available Plex libraries: {[(s.title, s.type) for s in sections]}")
                except Exception as e:
                    print(f"[DEBUG] Error getting Plex libraries: {e}")
            else:
                print(f"[DEBUG] Plex server is NOT connected!")
        
        # Check Spotify client - EXACTLY like dashboard.py
        if not spotify_client:
            add_activity_item("❌", "Metadata Update", "Spotify client not available", "Now")
            return jsonify({"success": False, "error": "Spotify client not available"}), 400
        
        # Reset state
        metadata_update_state.update({
            'status': 'running',
            'current_artist': 'Loading artists...',
            'processed': 0,
            'total': 0,
            'percentage': 0.0,
            'successful': 0,
            'failed': 0,
            'started_at': datetime.now(),
            'completed_at': None,
            'error': None,
            'refresh_interval_days': refresh_interval_days
        })
        
        # Start the metadata update worker - EXACTLY like dashboard.py
        def run_metadata_update():
            try:
                metadata_worker = WebMetadataUpdateWorker(
                    None,  # Artists will be loaded in the worker thread - EXACTLY like dashboard.py
                    media_client,
                    spotify_client,
                    active_server,
                    refresh_interval_days
                )
                metadata_worker.run()
            except Exception as e:
                print(f"Error in metadata update worker: {e}")
                metadata_update_state['status'] = 'error'
                metadata_update_state['error'] = str(e)
                add_activity_item("❌", "Metadata Error", str(e), "Now")
        
        metadata_update_worker = metadata_update_executor.submit(run_metadata_update)
        
        add_activity_item("🎵", "Metadata Update", "Loading artists from library...", "Now")
        
        return jsonify({"success": True})
        
    except Exception as e:
        print(f"Error starting metadata update: {e}")
        metadata_update_state['status'] = 'error'
        metadata_update_state['error'] = str(e)
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/metadata/stop', methods=['POST'])
def stop_metadata_update():
    """Stop the metadata update process"""
    global metadata_update_state
    
    try:
        if metadata_update_state['status'] == 'running':
            metadata_update_state['status'] = 'stopping'
            metadata_update_state['current_artist'] = 'Stopping...'
            add_activity_item("⏹️", "Metadata Update", "Stopping metadata update process", "Now")
        
        return jsonify({"success": True})
        
    except Exception as e:
        print(f"Error stopping metadata update: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/metadata/status', methods=['GET'])
def get_metadata_update_status():
    """Get current metadata update status"""
    try:
        # Return a copy of the state with datetime serialization
        state_copy = metadata_update_state.copy()
        
        # Convert datetime objects to ISO format for JSON serialization
        if state_copy.get('started_at'):
            state_copy['started_at'] = state_copy['started_at'].isoformat()
        if state_copy.get('completed_at'):
            state_copy['completed_at'] = state_copy['completed_at'].isoformat()
        
        return jsonify({"success": True, "status": state_copy})
        
    except Exception as e:
        print(f"Error getting metadata update status: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

@app.route('/api/active-media-server', methods=['GET'])
def get_active_media_server():
    """Get the currently active media server"""
    try:
        active_server = config_manager.get_active_media_server()
        return jsonify({"success": True, "active_server": active_server})
    except Exception as e:
        print(f"Error getting active media server: {e}")
        return jsonify({"success": False, "error": str(e)}), 500

# ================================= #
# BEATPORT API ENDPOINTS            #
# ================================= #

@app.route('/api/beatport/genres', methods=['GET'])
def get_beatport_genres():
    """Get current Beatport genres with images dynamically scraped from homepage"""
    try:
        logger.info("🔍 API request for Beatport genres")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get query parameters
        include_images = request.args.get('include_images', 'false').lower() == 'true'

        # Discover genres dynamically
        if include_images:
            logger.info("🖼️ Including genre images in response (slower)")
            genres = scraper.discover_genres_with_images(include_images=True)
        else:
            logger.info("📝 Returning genres without images (faster)")
            genres = scraper.discover_genres_from_homepage()

        logger.info(f"✅ Successfully discovered {len(genres)} Beatport genres")

        return jsonify({
            "success": True,
            "genres": genres,
            "count": len(genres),
            "includes_images": include_images
        })

    except Exception as e:
        logger.error(f"❌ Error fetching Beatport genres: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "genres": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/tracks', methods=['GET'])
def get_beatport_genre_tracks(genre_slug, genre_id):
    """Get tracks for a specific Beatport genre"""
    try:
        logger.info(f"🎵 API request for {genre_slug} genre tracks (ID: {genre_id})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get query parameters
        limit = int(request.args.get('limit', '100'))

        # Create genre dict for scraper
        genre = {
            'name': genre_slug.replace('-', ' ').title(),
            'slug': genre_slug,
            'id': genre_id
        }

        # Scrape tracks for this genre
        tracks = scraper.scrape_genre_charts(genre, limit=limit)

        logger.info(f"✅ Successfully scraped {len(tracks)} tracks for {genre_slug}")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "genre": genre,
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching tracks for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/chart/extract', methods=['POST'])
def extract_beatport_chart_tracks():
    """Extract tracks from a specific Beatport chart URL"""
    try:
        data = request.get_json()
        chart_url = data.get('chart_url')
        chart_name = data.get('chart_name', 'Unknown Chart')
        limit = int(data.get('limit', 100))

        if not chart_url:
            return jsonify({
                "success": False,
                "error": "chart_url is required",
                "tracks": [],
                "count": 0
            }), 400

        logger.info(f"🔍 API request to extract tracks from chart: {chart_name}")
        logger.info(f"🔗 Chart URL: {chart_url}")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Extract tracks from the specific chart URL
        tracks = scraper.extract_tracks_from_chart(chart_url, chart_name, limit)

        logger.info(f"✅ Successfully extracted {len(tracks)} tracks from chart: {chart_name}")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "chart_name": chart_name,
            "chart_url": chart_url,
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error extracting tracks from chart: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/top-10', methods=['GET'])
def get_beatport_genre_top_10(genre_slug, genre_id):
    """Get top 10 tracks for a specific Beatport genre"""
    try:
        logger.info(f"🔥 API request for {genre_slug} genre top 10 tracks (ID: {genre_id})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Create genre dict for scraper
        genre = {
            'name': genre_slug.replace('-', ' ').title(),
            'slug': genre_slug,
            'id': genre_id
        }

        # Scrape top 10 tracks for this genre
        tracks = scraper.scrape_genre_top_10(genre)

        logger.info(f"✅ Successfully scraped {len(tracks)} top 10 tracks for {genre_slug}")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "genre": genre,
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching top 10 tracks for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/releases-top-10', methods=['GET'])
def get_beatport_genre_releases_top_10(genre_slug, genre_id):
    """Get top 10 releases for a specific Beatport genre"""
    try:
        logger.info(f"📊 API request for {genre_slug} genre top 10 releases (ID: {genre_id})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Create genre dict for scraper
        genre = {
            'name': genre_slug.replace('-', ' ').title(),
            'slug': genre_slug,
            'id': genre_id
        }

        # Scrape top 10 releases for this genre
        releases = scraper.scrape_genre_releases(genre, limit=10)

        logger.info(f"✅ Successfully scraped {len(releases)} top 10 releases for {genre_slug}")

        return jsonify({
            "success": True,
            "tracks": releases,
            "genre": genre,
            "count": len(releases)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching top 10 releases for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/releases-top-100', methods=['GET'])
def get_beatport_genre_releases_top_100(genre_slug, genre_id):
    """Get top 100 releases for a specific Beatport genre"""
    try:
        logger.info(f"📊 API request for {genre_slug} genre top 100 releases (ID: {genre_id})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get query parameters
        limit = int(request.args.get('limit', '100'))

        # Create genre dict for scraper
        genre = {
            'name': genre_slug.replace('-', ' ').title(),
            'slug': genre_slug,
            'id': genre_id
        }

        # Scrape top releases for this genre
        releases = scraper.scrape_genre_releases(genre, limit=limit)

        logger.info(f"✅ Successfully scraped {len(releases)} top 100 releases for {genre_slug}")

        return jsonify({
            "success": True,
            "tracks": releases,
            "genre": genre,
            "count": len(releases)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching top 100 releases for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/staff-picks', methods=['GET'])
def get_beatport_genre_staff_picks(genre_slug, genre_id):
    """Get staff picks for a specific Beatport genre"""
    try:
        logger.info(f"⭐ API request for {genre_slug} genre staff picks (ID: {genre_id})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get query parameters
        limit = int(request.args.get('limit', '50'))

        # Create genre dict for scraper
        genre = {
            'name': genre_slug.replace('-', ' ').title(),
            'slug': genre_slug,
            'id': genre_id
        }

        # Scrape staff picks for this genre
        tracks = scraper.scrape_genre_staff_picks(genre, limit=limit)

        logger.info(f"✅ Successfully scraped {len(tracks)} staff picks for {genre_slug}")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "genre": genre,
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching staff picks for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/hype-top-10', methods=['GET'])
def get_beatport_genre_hype_top_10(genre_slug, genre_id):
    """Get hype top 10 tracks for a specific Beatport genre"""
    try:
        logger.info(f"🚀 API request for {genre_slug} genre hype top 10 (ID: {genre_id})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Create genre dict for scraper
        genre = {
            'name': genre_slug.replace('-', ' ').title(),
            'slug': genre_slug,
            'id': genre_id
        }

        # Scrape hype top 10 for this genre
        tracks = scraper.scrape_genre_hype_top_10(genre)

        logger.info(f"✅ Successfully scraped {len(tracks)} hype top 10 tracks for {genre_slug}")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "genre": genre,
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching hype top 10 for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/hype-top-100', methods=['GET'])
def get_beatport_genre_hype_top_100(genre_slug, genre_id):
    """Get hype top 100 tracks for a specific Beatport genre"""
    try:
        logger.info(f"💥 API request for {genre_slug} genre hype top 100 (ID: {genre_id})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Create genre dict for scraper
        genre = {
            'name': genre_slug.replace('-', ' ').title(),
            'slug': genre_slug,
            'id': genre_id
        }

        # Scrape hype top 100 for this genre
        tracks = scraper.scrape_genre_hype_charts(genre, limit=100)

        logger.info(f"✅ Successfully scraped {len(tracks)} hype top 100 tracks for {genre_slug}")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "genre": genre,
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching hype top 100 for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/hype-picks', methods=['GET'])
def get_beatport_genre_hype_picks(genre_slug, genre_id):
    """Get hype picks for a specific Beatport genre"""
    try:
        logger.info(f"⚡ API request for {genre_slug} genre hype picks (ID: {genre_id})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get query parameters
        limit = int(request.args.get('limit', '50'))

        # Create genre dict for scraper
        genre = {
            'name': genre_slug.replace('-', ' ').title(),
            'slug': genre_slug,
            'id': genre_id
        }

        # Scrape hype picks for this genre
        tracks = scraper.scrape_genre_hype_picks(genre, limit=limit)

        logger.info(f"✅ Successfully scraped {len(tracks)} hype picks for {genre_slug}")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "genre": genre,
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching hype picks for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/latest-releases', methods=['GET'])
def get_beatport_genre_latest_releases(genre_slug, genre_id):
    """Get latest releases for a specific Beatport genre"""
    try:
        logger.info(f"🕒 API request for {genre_slug} genre latest releases (ID: {genre_id})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get query parameters
        limit = int(request.args.get('limit', '50'))

        # Create genre dict for scraper
        genre = {
            'name': genre_slug.replace('-', ' ').title(),
            'slug': genre_slug,
            'id': genre_id
        }

        # Scrape latest releases for this genre
        tracks = scraper.scrape_genre_latest_releases(genre, limit=limit)

        logger.info(f"✅ Successfully scraped {len(tracks)} latest releases for {genre_slug}")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "genre": genre,
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching latest releases for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/new-charts', methods=['GET'])
def get_beatport_genre_new_charts(genre_slug, genre_id):
    """Get new charts for a specific Beatport genre"""
    try:
        logger.info(f"📈 API request for {genre_slug} genre new charts (ID: {genre_id})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get query parameters
        limit = int(request.args.get('limit', '50'))

        # Create genre dict for scraper
        genre = {
            'name': genre_slug.replace('-', ' ').title(),
            'slug': genre_slug,
            'id': genre_id
        }

        # Scrape new charts for this genre
        tracks = scraper.scrape_genre_new_charts(genre, limit=limit)

        logger.info(f"✅ Successfully scraped {len(tracks)} new charts for {genre_slug}")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "genre": genre,
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching new charts for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/hero', methods=['GET'])
def get_beatport_genre_hero(genre_slug, genre_id):
    """Get hero slider data for a specific Beatport genre with 1-hour caching"""
    try:
        logger.info(f"🎠 API request for {genre_slug} genre hero slider (ID: {genre_id})")

        # Check cache first (1-hour TTL like other genre data)
        cache_key = f"hero_{genre_slug}_{genre_id}"
        cached_data = get_cached_beatport_data('genre', cache_key, genre_slug)

        if cached_data:
            logger.info(f"💾 Returning cached hero data for {genre_slug}")
            return jsonify({
                "success": True,
                "releases": cached_data,
                "count": len(cached_data),
                "genre_slug": genre_slug,
                "genre_id": genre_id,
                "cached": True,
                "cache_timestamp": time.time()
            })

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Scrape hero slider data
        hero_releases = scraper.scrape_genre_hero_slider(genre_slug, genre_id)

        if hero_releases:
            # Cache the data (1-hour TTL)
            set_cached_beatport_data('genre', cache_key, hero_releases, genre_slug)

            logger.info(f"✅ Successfully scraped and cached {len(hero_releases)} hero releases for {genre_slug}")

            return jsonify({
                "success": True,
                "releases": hero_releases,
                "count": len(hero_releases),
                "genre_slug": genre_slug,
                "genre_id": genre_id,
                "cached": False,
                "scrape_timestamp": time.time()
            })
        else:
            logger.info(f"⚠️ No hero releases found for {genre_slug}")
            return jsonify({
                "success": False,
                "releases": [],
                "count": 0,
                "genre_slug": genre_slug,
                "genre_id": genre_id,
                "message": "No hero releases found"
            })

    except Exception as e:
        logger.error(f"❌ Error fetching hero data for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "releases": [],
            "count": 0,
            "genre_slug": genre_slug,
            "genre_id": genre_id
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/top-10-lists', methods=['GET'])
def get_beatport_genre_top10_lists(genre_slug, genre_id):
    """Get Top 10 lists (Beatport + Hype) for a specific genre with 1-hour caching"""
    try:
        logger.info(f"🎵 API request for {genre_slug} Top 10 lists (ID: {genre_id})")

        # Check cache first (1-hour TTL)
        cached_data = get_cached_beatport_data('genre', 'top_10_lists', genre_slug)
        if cached_data:
            logger.info(f"✅ Returning cached Top 10 lists for {genre_slug}")
            cached_data['success'] = True
            cached_data['cached'] = True
            return jsonify(cached_data)

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Scrape Top 10 lists from genre page
        top10_data = scraper.scrape_genre_top10_tracks(genre_slug, genre_id)

        if not top10_data['beatport_top10'] and not top10_data['hype_top10']:
            return jsonify({
                "success": False,
                "error": "No Top 10 tracks found for this genre",
                "beatport_top10": [],
                "hype_top10": [],
                "beatport_count": 0,
                "hype_count": 0,
                "has_hype_section": False,
                "genre_slug": genre_slug,
                "genre_id": genre_id,
                "cached": False
            })

        # Prepare response data
        response_data = {
            "beatport_top10": top10_data['beatport_top10'],
            "hype_top10": top10_data['hype_top10'],
            "beatport_count": len(top10_data['beatport_top10']),
            "hype_count": len(top10_data['hype_top10']),
            "has_hype_section": top10_data['has_hype_section'],
            "total_tracks": top10_data['total_tracks'],
            "genre_slug": genre_slug,
            "genre_id": genre_id,
            "cached": False,
            "cache_ttl": 3600  # 1 hour
        }

        # Cache the data (1-hour TTL)
        set_cached_beatport_data('genre', 'top_10_lists', response_data, genre_slug)

        logger.info(f"✅ Successfully fetched {response_data['beatport_count']} Beatport + {response_data['hype_count']} Hype Top 10 tracks for {genre_slug}")

        response_data['success'] = True
        return jsonify(response_data)

    except Exception as e:
        logger.error(f"❌ Error fetching Top 10 lists for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "beatport_top10": [],
            "hype_top10": [],
            "beatport_count": 0,
            "hype_count": 0,
            "has_hype_section": False,
            "genre_slug": genre_slug,
            "genre_id": genre_id,
            "cached": False
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/top-10-releases', methods=['GET'])
def get_beatport_genre_top10_releases(genre_slug, genre_id):
    """Get Top 10 releases for a specific genre using .partial-artwork elements with 1-hour caching"""
    try:
        logger.info(f"💿 API request for {genre_slug} Top 10 releases (ID: {genre_id})")

        # Check cache first (1-hour TTL)
        cached_data = get_cached_beatport_data('genre', 'top_10_releases', genre_slug)
        if cached_data:
            logger.info(f"✅ Returning cached Top 10 releases for {genre_slug}")
            cached_data['success'] = True
            cached_data['cached'] = True
            return jsonify(cached_data)

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Scrape Top 10 releases from genre page
        releases = scraper.scrape_genre_top10_releases(genre_slug, genre_id)

        if not releases:
            return jsonify({
                "success": False,
                "error": "No Top 10 releases found for this genre",
                "releases": [],
                "releases_count": 0,
                "genre_slug": genre_slug,
                "genre_id": genre_id,
                "cached": False
            })

        # Prepare response data
        response_data = {
            "releases": releases,
            "releases_count": len(releases),
            "genre_slug": genre_slug,
            "genre_id": genre_id,
            "cached": False,
            "cache_ttl": 3600  # 1 hour
        }

        # Cache the data (1-hour TTL)
        set_cached_beatport_data('genre', 'top_10_releases', response_data, genre_slug)

        logger.info(f"✅ Successfully fetched {response_data['releases_count']} Top 10 releases for {genre_slug}")

        response_data['success'] = True
        return jsonify(response_data)

    except Exception as e:
        logger.error(f"❌ Error fetching Top 10 releases for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "releases": [],
            "releases_count": 0,
            "genre_slug": genre_slug,
            "genre_id": genre_id,
            "cached": False
        }), 500

@app.route('/api/beatport/genre/<genre_slug>/<genre_id>/sections', methods=['GET'])
def get_beatport_genre_sections(genre_slug, genre_id):
    """Discover all available sections for a specific Beatport genre"""
    try:
        logger.info(f"🔍 API request for {genre_slug} genre sections discovery (ID: {genre_id})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Create genre dict for scraper
        genre = {
            'name': genre_slug.replace('-', ' ').title(),
            'slug': genre_slug,
            'id': genre_id
        }

        # Discover sections for this genre
        sections = scraper.discover_genre_page_sections(genre)

        logger.info(f"✅ Successfully discovered sections for {genre_slug}")

        return jsonify({
            "success": True,
            "sections": sections,
            "genre": genre
        })

    except Exception as e:
        logger.error(f"❌ Error discovering sections for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "sections": {}
        }), 500

@app.route('/api/beatport/top-100', methods=['GET'])
def get_beatport_top_100():
    """Get Beatport Top 100 tracks"""
    try:
        logger.info("🔥 API request for Beatport Top 100")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get query parameters
        limit = int(request.args.get('limit', '100'))

        # Scrape Top 100
        tracks = scraper.scrape_top_100(limit=limit)

        logger.info(f"✅ Successfully scraped {len(tracks)} tracks from Beatport Top 100")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "chart_name": "Beatport Top 100",
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching Beatport Top 100: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/genre-image/<genre_slug>/<genre_id>', methods=['GET'])
def get_beatport_genre_image(genre_slug, genre_id):
    """Get image for a specific Beatport genre"""
    try:
        logger.info(f"🖼️ API request for {genre_slug} genre image")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Construct genre URL
        genre_url = f"{scraper.base_url}/genre/{genre_slug}/{genre_id}"

        # Get genre image
        image_url = scraper.get_genre_image(genre_url)

        if image_url:
            logger.info(f"✅ Found image for {genre_slug}")
            return jsonify({
                "success": True,
                "image_url": image_url,
                "genre_slug": genre_slug,
                "genre_id": genre_id
            })
        else:
            logger.info(f"⚠️ No image found for {genre_slug}")
            return jsonify({
                "success": False,
                "image_url": None,
                "genre_slug": genre_slug,
                "genre_id": genre_id
            })

    except Exception as e:
        logger.error(f"❌ Error fetching image for {genre_slug}: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "image_url": None
        }), 500

@app.route('/api/beatport/hype-top-100', methods=['GET'])
def get_beatport_hype_top_100():
    """Get Beatport Hype Top 100 - Improved with fixed URL"""
    try:
        logger.info("🔥 API request for Beatport Hype Top 100")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get query parameters
        limit = int(request.args.get('limit', '100'))

        # Scrape Hype Top 100 using improved method
        tracks = scraper.scrape_hype_top_100(limit=limit)

        logger.info(f"✅ Successfully scraped {len(tracks)} tracks from Beatport Hype Top 100")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "chart_name": "Beatport Hype Top 100",
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching Beatport Hype Top 100: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/top-100-releases', methods=['GET'])
def get_beatport_top_100_releases():
    """Get Beatport Top 100 Releases - New endpoint"""
    try:
        logger.info("📊 API request for Beatport Top 100 Releases")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get query parameters
        limit = int(request.args.get('limit', '100'))

        # Scrape Top 100 Releases using new method
        tracks = scraper.scrape_top_100_releases(limit=limit)

        logger.info(f"✅ Successfully scraped {len(tracks)} tracks from Beatport Top 100 Releases")

        return jsonify({
            "success": True,
            "tracks": tracks,
            "chart_name": "Top 100 New Releases",
            "count": len(tracks)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching Beatport Top 100 Releases: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "count": 0
        }), 500

@app.route('/api/beatport/homepage/new-releases', methods=['GET'])
def get_beatport_homepage_new_releases():
    """Get Beatport New Releases from homepage section"""
    try:
        limit = int(request.args.get('limit', 40))
        logger.info(f"🆕 API request for Beatport homepage New Releases (limit: {limit})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get new releases from homepage
        new_releases = scraper.scrape_new_releases(limit=limit)

        logger.info(f"✅ Successfully extracted {len(new_releases)} new releases from homepage")

        return jsonify({
            "success": True,
            "tracks": new_releases,
            "track_count": len(new_releases),
            "source": "beatport_homepage_new_releases"
        })

    except Exception as e:
        logger.error(f"❌ Error getting Beatport homepage new releases: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "track_count": 0
        }), 500

@app.route('/api/beatport/homepage/hype-picks', methods=['GET'])
def get_beatport_homepage_hype_picks():
    """Get Beatport Hype Picks from homepage section"""
    try:
        limit = int(request.args.get('limit', 40))
        logger.info(f"🔥 API request for Beatport homepage Hype Picks (limit: {limit})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get hype picks from homepage
        hype_picks = scraper.scrape_hype_picks_homepage(limit=limit)

        logger.info(f"✅ Successfully extracted {len(hype_picks)} hype picks from homepage")

        return jsonify({
            "success": True,
            "tracks": hype_picks,
            "track_count": len(hype_picks),
            "source": "beatport_homepage_hype_picks"
        })

    except Exception as e:
        logger.error(f"❌ Error getting Beatport homepage hype picks: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "track_count": 0
        }), 500

@app.route('/api/beatport/homepage/top-10-releases', methods=['GET'])
def get_beatport_homepage_top_10_releases():
    """Get Beatport Top 10 Releases from homepage section"""
    try:
        limit = int(request.args.get('limit', 10))
        logger.info(f"🔟 API request for Beatport homepage Top 10 Releases (limit: {limit})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get top 10 releases from homepage
        top_10_releases = scraper.scrape_top_10_releases_homepage(limit=limit)

        logger.info(f"✅ Successfully extracted {len(top_10_releases)} top 10 releases from homepage")

        return jsonify({
            "success": True,
            "tracks": top_10_releases,
            "track_count": len(top_10_releases),
            "source": "beatport_homepage_top_10_releases"
        })

    except Exception as e:
        logger.error(f"❌ Error getting Beatport homepage top 10 releases: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "track_count": 0
        }), 500

@app.route('/api/beatport/homepage/top-10-lists', methods=['GET'])
def get_beatport_homepage_top10_lists():
    """Get Beatport Top 10 Lists from homepage - both Beatport Top 10 and Hype Top 10"""
    try:
        logger.info("🏆 API request for Beatport homepage Top 10 Lists")

        # Check cache first
        cached_data = get_cached_beatport_data('homepage', 'top_10_lists')
        if cached_data:
            logger.info("🏆 Returning cached top 10 lists data")
            response = jsonify(cached_data)
            return add_cache_headers(response, 3600)  # 1 hour

        # Cache miss - scrape fresh data
        logger.info("🔄 Cache miss - scraping fresh top 10 lists data...")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get top 10 lists from homepage
        top10_lists = scraper.scrape_homepage_top10_lists()

        logger.info(f"✅ Successfully extracted Beatport Top 10: {len(top10_lists['beatport_top10'])}, Hype Top 10: {len(top10_lists['hype_top10'])}")

        # Prepare response data
        response_data = {
            "success": True,
            "beatport_top10": top10_lists["beatport_top10"],
            "hype_top10": top10_lists["hype_top10"],
            "beatport_count": len(top10_lists["beatport_top10"]),
            "hype_count": len(top10_lists["hype_top10"]),
            "source": "beatport_homepage_top10_lists"
        }

        # Cache the successful response
        set_cached_beatport_data('homepage', 'top_10_lists', response_data)

        response = jsonify(response_data)
        return add_cache_headers(response, 3600)  # 1 hour

    except Exception as e:
        logger.error(f"❌ Error getting Beatport homepage top 10 lists: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "beatport_top10": [],
            "hype_top10": [],
            "beatport_count": 0,
            "hype_count": 0
        }), 500

@app.route('/api/beatport/homepage/top-10-releases-cards', methods=['GET'])
def get_beatport_homepage_top10_releases_cards():
    """Get Beatport Top 10 Releases CARDS from homepage (not individual tracks)"""
    try:
        logger.info("💿 API request for Beatport homepage Top 10 Releases CARDS")

        # Check cache first
        cached_data = get_cached_beatport_data('homepage', 'top_10_releases')
        if cached_data:
            logger.info("💿 Returning cached top 10 releases data")
            response = jsonify(cached_data)
            return add_cache_headers(response, 3600)  # 1 hour

        # Cache miss - scrape fresh data
        logger.info("🔄 Cache miss - scraping fresh top 10 releases data...")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get top 10 releases from homepage
        top10_releases = scraper.scrape_homepage_top10_releases()

        logger.info(f"✅ API extracted {len(top10_releases)} Top 10 Release Cards")

        # Debug: Log first release if any
        if top10_releases:
            logger.info(f"First release: {top10_releases[0].get('title', 'No title')} by {top10_releases[0].get('artist', 'No artist')}")
        else:
            logger.warning("❌ No releases found by scraper")

        # Prepare response data
        response_data = {
            "success": True,
            "releases": top10_releases,
            "releases_count": len(top10_releases),
            "source": "beatport_homepage_top10_releases_cards"
        }

        # Cache the successful response
        set_cached_beatport_data('homepage', 'top_10_releases', response_data)

        response = jsonify(response_data)
        return add_cache_headers(response, 3600)  # 1 hour

    except Exception as e:
        logger.error(f"❌ Error getting Beatport homepage Top 10 Releases cards: {e}")
        import traceback
        logger.error(f"Full traceback: {traceback.format_exc()}")
        return jsonify({
            "success": False,
            "error": str(e),
            "releases": [],
            "releases_count": 0
        }), 500

@app.route('/api/beatport/scrape-releases', methods=['POST'])
def scrape_beatport_releases():
    """General scraper endpoint - takes release URLs and returns tracks"""
    try:
        data = request.get_json()
        if not data:
            return jsonify({
                "success": False,
                "error": "No JSON data provided",
                "tracks": [],
                "track_count": 0
            }), 400

        release_urls = data.get('release_urls', [])
        source_name = data.get('source_name', 'General Release Scraper')

        if not release_urls:
            return jsonify({
                "success": False,
                "error": "No release URLs provided",
                "tracks": [],
                "track_count": 0
            }), 400

        logger.info(f"🎯 API request to scrape {len(release_urls)} release URLs with source: {source_name}")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Use our new general scraper function
        tracks = scraper.scrape_multiple_releases(release_urls, source_name)

        logger.info(f"✅ Successfully extracted {len(tracks)} tracks from {len(release_urls)} releases")

        # Apply text cleaning to track data
        cleaned_tracks = []
        for track in tracks:
            cleaned_track = track.copy()
            if 'title' in cleaned_track:
                cleaned_track['title'] = clean_beatport_text(cleaned_track['title'])
            if 'artist' in cleaned_track:
                cleaned_track['artist'] = clean_beatport_text(cleaned_track['artist'])
            if 'label' in cleaned_track:
                cleaned_track['label'] = clean_beatport_text(cleaned_track['label'])
            cleaned_tracks.append(cleaned_track)

        return jsonify({
            "success": True,
            "tracks": cleaned_tracks,
            "track_count": len(cleaned_tracks),
            "source": source_name,
            "release_urls_processed": len(release_urls)
        })

    except Exception as e:
        logger.error(f"❌ Error scraping releases: {e}")
        import traceback
        logger.error(f"Full traceback: {traceback.format_exc()}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "track_count": 0
        }), 500

@app.route('/api/beatport/homepage/featured-charts', methods=['GET'])
def get_beatport_homepage_featured_charts():
    """Get Beatport Featured Charts from homepage section"""
    try:
        limit = int(request.args.get('limit', 20))
        logger.info(f"📊 API request for Beatport homepage Featured Charts (limit: {limit})")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get featured charts from homepage
        featured_charts = scraper.scrape_featured_charts(limit=limit)

        logger.info(f"✅ Successfully extracted {len(featured_charts)} featured charts from homepage")

        return jsonify({
            "success": True,
            "tracks": featured_charts,
            "track_count": len(featured_charts),
            "source": "beatport_homepage_featured_charts"
        })

    except Exception as e:
        logger.error(f"❌ Error getting Beatport homepage featured charts: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "tracks": [],
            "track_count": 0
        }), 500

@app.route('/api/beatport/chart-sections', methods=['GET'])
def get_beatport_chart_sections():
    """Get dynamically discovered Beatport chart sections"""
    try:
        logger.info("🔍 API request for Beatport chart sections discovery")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Discover chart sections dynamically
        chart_sections = scraper.discover_chart_sections()

        logger.info(f"✅ Successfully discovered chart sections")

        return jsonify({
            "success": True,
            "chart_sections": chart_sections,
            "summary": chart_sections.get('summary', {})
        })

    except Exception as e:
        logger.error(f"❌ Error discovering Beatport chart sections: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "chart_sections": {},
            "summary": {}
        }), 500

@app.route('/api/beatport/dj-charts-improved', methods=['GET'])
def get_beatport_dj_charts_improved():
    """Get Beatport DJ Charts using improved method"""
    try:
        logger.info("🎧 API request for Beatport DJ Charts (improved)")

        # Initialize the Beatport scraper
        scraper = BeatportUnifiedScraper()

        # Get query parameters
        limit = int(request.args.get('limit', '20'))

        # Scrape DJ Charts using improved method
        charts = scraper.scrape_dj_charts(limit=limit)

        logger.info(f"✅ Successfully scraped {len(charts)} DJ charts")

        return jsonify({
            "success": True,
            "charts": charts,
            "chart_name": "Beatport DJ Charts",
            "count": len(charts)
        })

    except Exception as e:
        logger.error(f"❌ Error fetching Beatport DJ Charts: {e}")
        return jsonify({
            "success": False,
            "error": str(e),
            "charts": [],
            "count": 0
        }), 500


@app.route('/api/beatport/hype-picks')
def get_beatport_hype_picks():
    """Get Beatport Hype Picks for the rebuild slider grid (EXACT same pattern as new-releases)"""
    try:
        logger.info("🔥 Fetching Beatport hype picks...")

        # Check cache first
        cached_data = get_cached_beatport_data('homepage', 'hype_picks')
        if cached_data:
            logger.info("🔥 Returning cached hype picks data")
            response = jsonify(cached_data)
            return add_cache_headers(response, 3600)  # 1 hour

        # Cache miss - scrape fresh data
        logger.info("🔄 Cache miss - scraping fresh hype picks data...")

        # Initialize scraper
        scraper = BeatportUnifiedScraper()

        # Get page and extract releases
        soup = scraper.get_page(scraper.base_url)
        if not soup:
            raise Exception("Could not fetch Beatport homepage")

        # Extract hype pick cards using data-testid selector (equivalent to new-releases CSS selector)
        hype_pick_cards = soup.select('[data-testid="hype-picks"]')
        releases = []

        logger.info(f"🔍 Found {len(hype_pick_cards)} hype pick cards")

        for i, card in enumerate(hype_pick_cards[:100]):  # Limit to 100 for 10 slides (same as new-releases)
            release_data = {}

            # Extract title (exact same logic as new-releases)
            title_elem = card.select_one('[class*="title"], [class*="Title"], h1, h2, h3, h4, h5, h6')
            if title_elem:
                title_text = title_elem.get_text(strip=True)
                if title_text and len(title_text) > 2 and title_text not in ['Hype Picks', 'Buy', 'Play']:
                    release_data['title'] = title_text

            # Extract artist (exact same logic as new-releases)
            artist_elem = card.select_one('[class*="artist"], [class*="Artist"], a[href*="/artist/"]')
            if artist_elem:
                artist_text = artist_elem.get_text(strip=True)
                if artist_text and len(artist_text) > 1:
                    release_data['artist'] = artist_text

            # Extract label (exact same logic as new-releases)
            label_elem = card.select_one('[class*="label"], [class*="Label"], a[href*="/label/"]')
            if label_elem:
                label_text = label_elem.get_text(strip=True)
                if label_text and len(label_text) > 1:
                    release_data['label'] = label_text

            # Extract URL (exact same logic as new-releases)
            url_link = card.select_one('a[href*="/release/"]')
            if url_link:
                href = url_link.get('href')
                if href:
                    release_data['url'] = urljoin(scraper.base_url, href)

            # Extract image (exact same logic as new-releases)
            img = card.select_one('img')
            if img:
                src = img.get('src') or img.get('data-src') or img.get('data-lazy-src')
                if src:
                    release_data['image_url'] = src

            # URL fallback for title (exact same logic as new-releases)
            if not release_data.get('title') and release_data.get('url'):
                url_parts = release_data['url'].split('/release/')
                if len(url_parts) > 1:
                    slug = url_parts[1].split('/')[0]
                    release_data['title'] = slug.replace('-', ' ').title()

            # Only add if we have essential data (exact same logic as new-releases)
            if release_data.get('title') and release_data.get('url'):
                # Add fallbacks for missing data (exact same logic as new-releases)
                if not release_data.get('artist'):
                    release_data['artist'] = 'Various Artists'
                if not release_data.get('label'):
                    release_data['label'] = 'Unknown Label'

                releases.append(release_data)

        logger.info(f"✅ Successfully extracted {len(releases)} hype picks")

        # Prepare response data
        response_data = {
            'success': True,
            'releases': releases,
            'count': len(releases),
            'slides': (len(releases) + 9) // 10,  # Calculate number of slides needed (same as new-releases)
            'timestamp': datetime.now().isoformat()
        }

        # Cache the successful response
        set_cached_beatport_data('homepage', 'hype_picks', response_data)

        response = jsonify(response_data)
        return add_cache_headers(response, 3600)  # 1 hour

    except Exception as e:
        logger.error(f"❌ Error getting Beatport hype picks: {e}")
        return jsonify({
            'success': False,
            'error': str(e),
            'releases': [],
            'count': 0
        }), 500


@app.route('/api/beatport/discovery/start/<url_hash>', methods=['POST'])
def start_beatport_discovery(url_hash):
    """Start Spotify discovery for Beatport chart tracks"""
    import json
    try:
        logger.info(f"🔍 Starting Beatport discovery for: {url_hash}")

        # Get chart data from request body
        data = request.get_json() or {}
        print(f"🔍 Raw request data: {data}")

        chart_data = data.get('chart_data')
        print(f"🔍 Chart data extracted: {chart_data is not None}")

        # Debug logging
        if chart_data:
            print(f"🔍 Chart data keys: {list(chart_data.keys()) if isinstance(chart_data, dict) else 'Not a dict'}")
            print(f"🔍 Chart name: {chart_data.get('name') if isinstance(chart_data, dict) else 'N/A'}")
            if isinstance(chart_data, dict) and 'tracks' in chart_data:
                print(f"🔍 Number of tracks: {len(chart_data['tracks'])}")
                if chart_data['tracks']:
                    print(f"🔍 First track: {chart_data['tracks'][0]}")
        else:
            print("🔍 No chart data received")

        if not chart_data or not chart_data.get('tracks'):
            return jsonify({"error": "Chart data with tracks is required"}), 400

        # Initialize Beatport chart state (similar to YouTube)
        if url_hash not in beatport_chart_states:
            beatport_chart_states[url_hash] = {
                'chart': chart_data,
                'phase': 'fresh',
                'discovery_results': [],
                'discovery_progress': 0,
                'spotify_matches': 0,
                'spotify_total': len(chart_data['tracks']),
                'status': 'fresh',
                'last_accessed': time.time()
            }

        state = beatport_chart_states[url_hash]
        state['last_accessed'] = time.time()

        if state['phase'] == 'discovering':
            return jsonify({"error": "Discovery already in progress"}), 400

        # Update phase to discovering
        state['phase'] = 'discovering'
        state['status'] = 'discovering'
        state['discovery_progress'] = 0
        state['spotify_matches'] = 0

        # Add activity for discovery start
        chart_name = chart_data.get('name', 'Unknown Chart')
        track_count = len(chart_data['tracks'])
        add_activity_item("🔍", "Beatport Discovery Started", f"'{chart_name}' - {track_count} tracks", "Now")

        # Start discovery worker
        future = beatport_discovery_executor.submit(_run_beatport_discovery_worker, url_hash)
        state['discovery_future'] = future

        print(f"🔍 Started Spotify discovery for Beatport chart: {chart_name}")
        return jsonify({"success": True, "message": "Discovery started", "status": "discovering"})

    except Exception as e:
        logger.error(f"❌ Error starting Beatport discovery: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/beatport/discovery/status/<url_hash>', methods=['GET'])
def get_beatport_discovery_status(url_hash):
    """Get real-time discovery status for a Beatport chart"""
    try:
        if url_hash not in beatport_chart_states:
            return jsonify({"error": "Beatport chart not found"}), 404

        state = beatport_chart_states[url_hash]
        state['last_accessed'] = time.time()

        response = {
            'phase': state['phase'],
            'status': state['status'],
            'progress': state['discovery_progress'],
            'spotify_matches': state['spotify_matches'],
            'spotify_total': state['spotify_total'],
            'results': state['discovery_results'],
            'complete': state['phase'] == 'discovered'
        }

        return jsonify(response)

    except Exception as e:
        logger.error(f"❌ Error getting Beatport discovery status: {e}")
        return jsonify({"error": str(e)}), 500


@app.route('/api/beatport/discovery/update_match', methods=['POST'])
def update_beatport_discovery_match():
    """Update a Beatport discovery result with manually selected Spotify track"""
    try:
        data = request.get_json()
        identifier = data.get('identifier')  # url_hash
        track_index = data.get('track_index')
        spotify_track = data.get('spotify_track')

        if not identifier or track_index is None or not spotify_track:
            return jsonify({'error': 'Missing required fields'}), 400

        # Get the state
        state = beatport_chart_states.get(identifier)

        if not state:
            return jsonify({'error': 'Discovery state not found'}), 404

        if track_index >= len(state['discovery_results']):
            return jsonify({'error': 'Invalid track index'}), 400

        # Update the result
        result = state['discovery_results'][track_index]
        old_status = result.get('status')

        # Update with user-selected track
        result['status'] = '✅ Found'
        result['status_class'] = 'found'
        result['spotify_track'] = spotify_track['name']
        result['spotify_artist'] = ', '.join(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else spotify_track['artists']
        result['spotify_album'] = spotify_track['album']
        result['spotify_id'] = spotify_track['id']

        # Format duration (Beatport doesn't show duration in table, but store it anyway)
        duration_ms = spotify_track.get('duration_ms', 0)
        if duration_ms:
            minutes = duration_ms // 60000
            seconds = (duration_ms % 60000) // 1000
            result['duration'] = f"{minutes}:{seconds:02d}"
        else:
            result['duration'] = '0:00'

        # IMPORTANT: Also set spotify_data for sync/download compatibility
        result['spotify_data'] = {
            'id': spotify_track['id'],
            'name': spotify_track['name'],
            'artists': spotify_track['artists'],
            'album': spotify_track['album'],
            'duration_ms': spotify_track.get('duration_ms', 0)
        }

        result['manual_match'] = True  # Flag for tracking

        # Update match count if status changed from not found/error
        if old_status != 'found' and old_status != '✅ Found':
            state['spotify_matches'] = state.get('spotify_matches', 0) + 1

        logger.info(f"✅ Manual match updated: beatport - {identifier} - track {track_index}")
        logger.info(f"   → {result['spotify_artist']} - {result['spotify_track']}")

        return jsonify({'success': True, 'result': result})

    except Exception as e:
        logger.error(f"❌ Error updating Beatport discovery match: {e}")
        return jsonify({'error': str(e)}), 500


def clean_beatport_text(text):
    """Clean Beatport track/artist text for proper spacing"""
    if not text:
        return text

    import re
    # Fix common spacing issues
    text = re.sub(r'([a-z$!@#%&*])([A-Z])', r'\1 \2', text)  # Add space between lowercase/symbols and uppercase
    text = re.sub(r'([a-zA-Z]),([a-zA-Z])', r'\1, \2', text)  # Add space after comma
    text = re.sub(r'([a-zA-Z])(Mix|Remix|Extended|Version)\b', r'\1 \2', text)  # Fix mix types
    text = re.sub(r'\s+', ' ', text)  # Collapse multiple spaces
    text = text.strip()

    return text

def _run_beatport_discovery_worker(url_hash):
    """Background worker for Beatport discovery process (Spotify preferred, iTunes fallback)"""
    try:
        state = beatport_chart_states[url_hash]
        chart = state['chart']
        tracks = chart['tracks']

        # Determine which provider to use
        use_spotify = spotify_client and spotify_client.is_spotify_authenticated()
        discovery_source = 'spotify' if use_spotify else 'itunes'

        # Initialize iTunes client if needed
        itunes_client_instance = None
        if not use_spotify:
            from core.itunes_client import iTunesClient
            itunes_client_instance = iTunesClient()

        print(f"🔍 Starting {discovery_source.upper()} discovery for {len(tracks)} Beatport tracks...")

        # Store discovery source in state for frontend
        state['discovery_source'] = discovery_source

        # Process each track for discovery
        for i, track in enumerate(tracks):
            try:
                # Update progress
                state['discovery_progress'] = int((i / len(tracks)) * 100)

                # Get track info from Beatport data (frontend sends 'name' and 'artists' fields)
                track_title = clean_beatport_text(track.get('name', 'Unknown Title'))
                track_artists = track.get('artists', ['Unknown Artist'])
                # Handle artists - could be a list or string
                if isinstance(track_artists, list):
                    if len(track_artists) > 0 and isinstance(track_artists[0], str):
                        # Handle case like ["CID,Taylr Renee"] - split on comma and clean
                        track_artist = clean_beatport_text(track_artists[0].split(',')[0].strip())
                    else:
                        track_artist = clean_beatport_text(track_artists[0] if track_artists else 'Unknown Artist')
                else:
                    track_artist = clean_beatport_text(str(track_artists))

                print(f"🔍 Searching {discovery_source.upper()} for: '{track_artist}' - '{track_title}'")

                # Check discovery cache first
                cache_key = _get_discovery_cache_key(track_title, track_artist)
                try:
                    cache_db = get_database()
                    cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source)
                    if cached_match and _validate_discovery_cache_artist(track_artist, cached_match):
                        print(f"⚡ CACHE HIT [{i+1}/{len(tracks)}]: {track_artist} - {track_title}")
                        # Convert artists from ['str'] to [{'name': 'str'}] for Beatport frontend format
                        beatport_artists = cached_match.get('artists', [])
                        if beatport_artists and isinstance(beatport_artists[0], str):
                            cached_match['artists'] = [{'name': a} for a in beatport_artists]
                        result_entry = {
                            'index': i,
                            'beatport_track': {
                                'title': track_title,
                                'artist': track_artist
                            },
                            'status': 'found',
                            'status_class': 'found',
                            'discovery_source': discovery_source,
                            'spotify_data': cached_match
                        }
                        state['spotify_matches'] += 1
                        state['discovery_results'].append(result_entry)
                        continue
                except Exception as cache_err:
                    print(f"⚠️ Cache lookup error: {cache_err}")

                # Use matching engine for track matching
                found_track = None
                best_confidence = 0.0
                best_raw_track = None
                min_confidence = 0.9  # Higher threshold for Beatport to avoid bad matches

                # Generate search queries using matching engine (with fallback)
                try:
                    temp_track = type('TempTrack', (), {
                        'name': track_title,
                        'artists': [track_artist],
                        'album': None
                    })()
                    search_queries = matching_engine.generate_download_queries(temp_track)
                    print(f"🔍 Generated {len(search_queries)} search queries using matching engine")
                except Exception as e:
                    print(f"⚠️ Matching engine failed for Beatport, falling back to basic queries: {e}")
                    if use_spotify:
                        search_queries = [
                            f"{track_artist} {track_title}",
                            f'artist:"{track_artist}" track:"{track_title}"',
                            f'"{track_artist}" "{track_title}"'
                        ]
                    else:
                        search_queries = [
                            f"{track_artist} {track_title}",
                            f"{track_title} {track_artist}",
                            track_title
                        ]

                for query_idx, search_query in enumerate(search_queries):
                    try:
                        print(f"🔍 Query {query_idx + 1}/{len(search_queries)}: {search_query} ({discovery_source.upper()})")

                        raw_results = None
                        search_results = None

                        if use_spotify:
                            raw_results = spotify_client.sp.search(q=search_query, type='track', limit=10)
                            if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']:
                                continue
                            search_results = spotify_client.search_tracks(search_query, limit=10)
                        else:
                            search_results = itunes_client_instance.search_tracks(search_query, limit=10)

                        if not search_results:
                            continue

                        # Score all results using the matching engine
                        match, confidence, match_idx = _discovery_score_candidates(
                            track_title, track_artist, 0, search_results
                        )

                        if match and confidence > best_confidence and confidence >= min_confidence:
                            best_confidence = confidence
                            found_track = match
                            if use_spotify and raw_results:
                                best_raw_track = raw_results['tracks']['items'][match_idx] if match_idx < len(raw_results['tracks']['items']) else None
                            else:
                                best_raw_track = None
                            print(f"✅ New best Beatport match: {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

                        if best_confidence >= 0.9:
                            print(f"🎯 High confidence match found ({best_confidence:.3f}), stopping search")
                            break

                    except Exception as e:
                        print(f"❌ Error in {discovery_source.upper()} search for query '{search_query}': {e}")
                        continue

                # Strategy 4: Extended search with higher limit (last resort)
                if not found_track:
                    print(f"🔄 Beatport Strategy 4: Extended search with limit=50")
                    query = f"{track_artist} {track_title}"
                    if use_spotify:
                        extended_results = spotify_client.search_tracks(query, limit=50)
                    else:
                        extended_results = itunes_client_instance.search_tracks(query, limit=50)
                    if extended_results:
                        match, confidence, _ = _discovery_score_candidates(
                            track_title, track_artist, 0, extended_results
                        )
                        if match and confidence >= min_confidence:
                            found_track = match
                            best_confidence = confidence
                            print(f"✅ Strategy 4 Beatport match (extended): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")

                if found_track:
                    print(f"✅ Final Beatport match: {found_track.artists[0]} - {found_track.name} (confidence: {best_confidence:.3f})")
                else:
                    print(f"❌ No suitable match found (best confidence was {best_confidence:.3f}, required {min_confidence:.3f})")

                # Create result entry
                result_entry = {
                    'index': i,  # Add index for frontend table row identification
                    'beatport_track': {
                        'title': track_title,
                        'artist': track_artist
                    },
                    'status': 'found' if found_track else 'not_found',
                    'status_class': 'found' if found_track else 'not-found',
                    'discovery_source': discovery_source,
                    'confidence': best_confidence
                }

                if found_track:
                    if use_spotify:
                        # SPOTIFY result formatting
                        # Debug: show available attributes
                        print(f"🔍 Spotify track attributes: {dir(found_track)}")

                        # Format artists correctly for frontend compatibility
                        formatted_artists = []
                        if isinstance(found_track.artists, list):
                            # If it's already a list of strings, convert to objects with 'name' property
                            for artist in found_track.artists:
                                if isinstance(artist, str):
                                    formatted_artists.append({'name': artist})
                                else:
                                    # If it's already an object, use as-is
                                    formatted_artists.append(artist)
                        else:
                            # Single artist case
                            formatted_artists = [{'name': str(found_track.artists)}]

                        # Use full album object from raw Spotify data if available
                        album_data = best_raw_track.get('album', {}) if best_raw_track else {}
                        if not album_data:
                            # Fallback to string album name
                            album_data = {'name': found_track.album, 'album_type': 'album', 'images': []}

                        result_entry['spotify_data'] = {
                            'name': found_track.name,
                            'artists': formatted_artists,  # Now formatted as list of objects with 'name' property
                            'album': album_data,  # Full album object with images
                            'id': found_track.id,
                            'source': 'spotify'
                        }
                    else:
                        # ITUNES result formatting
                        # Note: iTunes Track dataclass has 'artists' (list) and 'image_url', not 'artist' and 'artwork_url'
                        result_artists = found_track.artists if hasattr(found_track, 'artists') else []
                        result_artist = result_artists[0] if result_artists else 'Unknown'
                        result_name = found_track.name if hasattr(found_track, 'name') else 'Unknown'
                        album_name = found_track.album if hasattr(found_track, 'album') else 'Unknown Album'
                        image_url = found_track.image_url if hasattr(found_track, 'image_url') else ''
                        track_id = found_track.id if hasattr(found_track, 'id') else ''

                        # Format artists as list of objects for frontend compatibility
                        formatted_artists = [{'name': result_artist}]

                        # Build album data with artwork
                        album_data = {
                            'name': album_name,
                            'album_type': 'album',
                            'images': [{'url': image_url, 'height': 300, 'width': 300}] if image_url else []
                        }

                        result_entry['spotify_data'] = {  # Use same key for frontend compatibility
                            'name': result_name,
                            'artists': formatted_artists,
                            'album': album_data,
                            'id': track_id,
                            'source': 'itunes'
                        }

                    state['spotify_matches'] += 1

                    # Save to discovery cache (normalize artists from [{name:str}] to [str] for canonical format)
                    if best_confidence >= 0.75:
                        try:
                            cache_data = dict(result_entry['spotify_data'])
                            cache_artists = cache_data.get('artists', [])
                            if cache_artists and isinstance(cache_artists[0], dict):
                                cache_data['artists'] = [a.get('name', '') for a in cache_artists]
                            cache_db = get_database()
                            cache_db.save_discovery_cache_match(
                                cache_key[0], cache_key[1], discovery_source, best_confidence,
                                cache_data, track_title, track_artist
                            )
                            print(f"💾 CACHE SAVED: {track_artist} - {track_title} (confidence: {best_confidence:.3f})")
                        except Exception as cache_err:
                            print(f"⚠️ Cache save error: {cache_err}")

                state['discovery_results'].append(result_entry)

                # Small delay to avoid rate limiting
                time.sleep(0.1)

            except Exception as e:
                print(f"❌ Error processing Beatport track {i}: {e}")
                # Add error result
                state['discovery_results'].append({
                    'index': i,  # Add index for frontend table row identification
                    'beatport_track': {
                        'title': track.get('name', 'Unknown'),  # Changed from 'title' to 'name' to match track structure
                        'artist': track.get('artists', ['Unknown'])[0] if isinstance(track.get('artists'), list) else 'Unknown'
                    },
                    'status': 'error',
                    'status_class': 'error',  # Add status class for CSS styling
                    'error': str(e),
                    'discovery_source': discovery_source
                })

        # Mark discovery as complete
        state['discovery_progress'] = 100
        state['phase'] = 'discovered'
        state['status'] = 'discovered'

        # Add activity for completion
        chart_name = chart.get('name', 'Unknown Chart')
        source_label = discovery_source.upper()
        add_activity_item("✅", f"Beatport Discovery Complete ({source_label})",
                         f"'{chart_name}' - {state['spotify_matches']}/{len(tracks)} tracks found", "Now")

        print(f"✅ Beatport discovery complete ({source_label}): {state['spotify_matches']}/{len(tracks)} tracks found")

    except Exception as e:
        print(f"❌ Error in Beatport discovery worker: {e}")
        if url_hash in beatport_chart_states:
            beatport_chart_states[url_hash]['status'] = 'error'
            beatport_chart_states[url_hash]['phase'] = 'fresh'

@app.route('/api/beatport/sync/start/<url_hash>', methods=['POST'])
def start_beatport_sync(url_hash):
    """Start sync process for a Beatport chart using discovered Spotify tracks"""
    try:
        print(f"🎧 Beatport sync start requested for: {url_hash}")

        if url_hash not in beatport_chart_states:
            print(f"❌ Beatport chart not found: {url_hash}")
            return jsonify({"error": "Beatport chart not found"}), 404

        state = beatport_chart_states[url_hash]
        state['last_accessed'] = time.time()  # Update access time

        print(f"🎧 Beatport chart state: phase={state.get('phase')}, has_discovery_results={len(state.get('discovery_results', []))}")

        if state['phase'] not in ['discovered', 'sync_complete']:
            print(f"❌ Beatport chart not ready for sync: {state['phase']}")
            return jsonify({"error": "Beatport chart not ready for sync"}), 400

        # Convert discovery results to Spotify tracks format
        spotify_tracks = convert_beatport_results_to_spotify_tracks(state['discovery_results'])

        if not spotify_tracks:
            return jsonify({"error": "No Spotify matches found for sync"}), 400

        # Create a temporary playlist ID for sync tracking
        sync_playlist_id = f"beatport_sync_{url_hash}_{int(time.time())}"

        # Initialize sync state
        state['sync_playlist_id'] = sync_playlist_id
        state['phase'] = 'syncing'
        state['sync_progress'] = {'status': 'starting', 'progress': 0}

        # Create sync job using existing infrastructure
        sync_data = {
            'id': sync_playlist_id,
            'name': state['chart']['name'],
            'tracks': spotify_tracks,
            'source': 'beatport',
            'source_id': url_hash
        }

        # Add to sync states using existing sync system
        with sync_lock:
            sync_states[sync_playlist_id] = {"status": "starting", "progress": {}}

        # Start sync in background using existing thread pool
        future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['name'], spotify_tracks)
        state['sync_future'] = future

        print(f"🎧 Started Beatport sync for chart: {state['chart']['name']}")
        return jsonify({"success": True, "sync_id": sync_playlist_id})

    except Exception as e:
        print(f"❌ Error starting Beatport sync: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/beatport/sync/status/<url_hash>', methods=['GET'])
def get_beatport_sync_status(url_hash):
    """Get sync status for a Beatport chart"""
    try:
        if url_hash not in beatport_chart_states:
            return jsonify({"error": "Beatport chart not found"}), 404

        state = beatport_chart_states[url_hash]
        state['last_accessed'] = time.time()  # Update access time
        sync_playlist_id = state.get('sync_playlist_id')

        if not sync_playlist_id:
            return jsonify({"error": "No sync process found"}), 404

        # Get sync status from sync states
        sync_state = sync_states.get(sync_playlist_id, {})

        response = {
            'status': sync_state.get('status', 'unknown'),
            'progress': sync_state.get('progress', {}),
            'sync_id': sync_playlist_id,
            'complete': sync_state.get('status') == 'finished',
            'error': sync_state.get('error')
        }

        # Check if sync completed successfully
        if sync_state.get('status') == 'finished':
            state['phase'] = 'sync_complete'
            # Extract playlist ID from sync result
            result = sync_state.get('result', {})
            state['converted_spotify_playlist_id'] = result.get('spotify_playlist_id')
            chart_name = state.get('chart', {}).get('name', 'Unknown Chart')
            add_activity_item("🔄", "Sync Complete", f"Beatport chart '{chart_name}' synced successfully", "Now")
        elif sync_state.get('status') == 'error':
            state['phase'] = 'discovered'  # Revert on error
            chart_name = state.get('chart', {}).get('name', 'Unknown Chart')
            add_activity_item("❌", "Sync Failed", f"Beatport chart '{chart_name}' sync failed", "Now")

        return jsonify(response)

    except Exception as e:
        print(f"❌ Error getting Beatport sync status: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/beatport/sync/cancel/<url_hash>', methods=['POST'])
def cancel_beatport_sync(url_hash):
    """Cancel sync for a Beatport chart"""
    try:
        if url_hash not in beatport_chart_states:
            return jsonify({"error": "Beatport chart not found"}), 404

        state = beatport_chart_states[url_hash]
        state['last_accessed'] = time.time()  # Update access time
        sync_playlist_id = state.get('sync_playlist_id')

        if sync_playlist_id and sync_playlist_id in sync_states:
            # Cancel the sync using existing sync infrastructure
            with sync_lock:
                sync_states[sync_playlist_id] = {"status": "cancelled"}

            # Cancel future if still running
            if 'sync_future' in state and state['sync_future']:
                state['sync_future'].cancel()

        # Revert Beatport state
        state['phase'] = 'discovered'
        state['sync_playlist_id'] = None
        state['sync_progress'] = {}

        print(f"🎧 Cancelled Beatport sync for: {url_hash}")
        return jsonify({"success": True})

    except Exception as e:
        print(f"❌ Error cancelling Beatport sync: {e}")
        return jsonify({"error": str(e)}), 500

# ===================================================================
# BEATPORT CHART PERSISTENCE API ENDPOINTS
# ===================================================================

@app.route('/api/beatport/charts', methods=['GET'])
def get_beatport_charts():
    """Get all persistent Beatport chart states for frontend hydration"""
    try:
        charts = []
        current_time = time.time()

        # Clean up old charts (older than 24 hours)
        to_remove = []
        for chart_hash, state in beatport_chart_states.items():
            last_accessed = state.get('last_accessed', 0)
            if current_time - last_accessed > 86400:  # 24 hours
                to_remove.append(chart_hash)
            else:
                # Include in response
                chart_info = {
                    'hash': chart_hash,
                    'name': state['chart']['name'],
                    'track_count': len(state['chart']['tracks']),
                    'phase': state.get('phase', 'fresh'),
                    'discovery_progress': state.get('discovery_progress', 0),
                    'spotify_matches': state.get('spotify_matches', 0),
                    'spotify_total': state.get('spotify_total', 0),
                    'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'),
                    'download_process_id': state.get('download_process_id'),
                    'last_accessed': last_accessed,
                    'chart_data': state['chart']  # Full chart data for restoration
                }
                charts.append(chart_info)

        # Remove old charts
        for chart_hash in to_remove:
            del beatport_chart_states[chart_hash]
            logger.info(f"🧹 Cleaned up old Beatport chart: {chart_hash}")

        logger.info(f"📊 Returning {len(charts)} Beatport charts for hydration")
        return jsonify(charts)

    except Exception as e:
        logger.error(f"❌ Error getting Beatport charts: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/beatport/charts/status/<chart_hash>', methods=['GET'])
def get_beatport_chart_status(chart_hash):
    """Get individual Beatport chart status with full state data"""
    try:
        if chart_hash not in beatport_chart_states:
            return jsonify({"error": "Beatport chart not found"}), 404

        state = beatport_chart_states[chart_hash]
        state['last_accessed'] = time.time()  # Update access time

        # Return full state including discovery results for modal restoration
        response = {
            'hash': chart_hash,
            'phase': state.get('phase', 'fresh'),
            'status': state.get('status', 'fresh'),
            'discovery_progress': state.get('discovery_progress', 0),
            'spotify_matches': state.get('spotify_matches', 0),
            'spotify_total': state.get('spotify_total', 0),
            'discovery_results': state.get('discovery_results', []),
            'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'),
            'download_process_id': state.get('download_process_id'),
            'sync_playlist_id': state.get('sync_playlist_id'),
            'sync_progress': state.get('sync_progress', {}),
            'chart_data': state['chart']  # Full chart data
        }

        return jsonify(response)

    except Exception as e:
        logger.error(f"❌ Error getting Beatport chart status: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/beatport/charts/update-phase/<chart_hash>', methods=['POST'])
def update_beatport_chart_phase(chart_hash):
    """Update Beatport chart phase (for modal close operations and reset)"""
    try:
        if chart_hash not in beatport_chart_states:
            return jsonify({"error": "Beatport chart not found"}), 404

        data = request.get_json() or {}
        new_phase = data.get('phase')
        is_reset = data.get('reset', False)

        if not new_phase:
            return jsonify({"error": "Phase is required"}), 400

        state = beatport_chart_states[chart_hash]
        state['phase'] = new_phase
        state['last_accessed'] = time.time()

        # Handle reset operation - clear discovery data
        if is_reset and new_phase == 'fresh':
            state['discovery_results'] = []
            state['discovery_progress'] = 0
            state['spotify_matches'] = 0
            state['status'] = 'fresh'
            state['converted_spotify_playlist_id'] = None
            state['download_process_id'] = None
            state['sync_playlist_id'] = None
            state['sync_progress'] = {}
            logger.info(f"🎧 Reset Beatport chart {chart_hash} to fresh state")
        else:
            # Handle other phase updates (like download phase transitions)
            converted_playlist_id = data.get('converted_spotify_playlist_id')
            if converted_playlist_id:
                state['converted_spotify_playlist_id'] = converted_playlist_id

            download_process_id = data.get('download_process_id')
            if download_process_id:
                state['download_process_id'] = download_process_id

        logger.info(f"🎧 Updated Beatport chart {chart_hash} phase to: {new_phase}")
        return jsonify({"success": True, "phase": new_phase})

    except Exception as e:
        logger.error(f"❌ Error updating Beatport chart phase: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/beatport/charts/delete/<chart_hash>', methods=['DELETE'])
def delete_beatport_chart(chart_hash):
    """Delete a Beatport chart from backend storage"""
    try:
        if chart_hash not in beatport_chart_states:
            return jsonify({"error": "Beatport chart not found"}), 404

        chart_name = beatport_chart_states[chart_hash]['chart']['name']
        del beatport_chart_states[chart_hash]

        logger.info(f"🗑️ Deleted Beatport chart: {chart_name}")
        return jsonify({"success": True, "message": f"Deleted chart: {chart_name}"})

    except Exception as e:
        logger.error(f"❌ Error deleting Beatport chart: {e}")
        return jsonify({"error": str(e)}), 500

# ── Mirrored Playlists ────────────────────────────────────────────────

@app.route('/api/mirror-playlist', methods=['POST'])
def mirror_playlist_endpoint():
    """Save or update a mirrored playlist."""
    try:
        data = request.get_json()
        if not data:
            return jsonify({"error": "No data received"}), 400

        source = data.get('source')
        source_playlist_id = data.get('source_playlist_id')
        name = data.get('name')
        tracks = data.get('tracks', [])

        if not all([source, source_playlist_id, name]):
            return jsonify({"error": "source, source_playlist_id, and name are required"}), 400

        database = get_database()
        profile_id = get_current_profile_id()

        playlist_id = database.mirror_playlist(
            source=source,
            source_playlist_id=str(source_playlist_id),
            name=name,
            tracks=tracks,
            profile_id=profile_id,
            description=data.get('description'),
            owner=data.get('owner'),
            image_url=data.get('image_url')
        )

        if playlist_id is None:
            return jsonify({"error": "Failed to mirror playlist"}), 500

        try:
            if automation_engine:
                automation_engine.emit('mirrored_playlist_created', {
                    'playlist_name': name,
                    'source': source,
                    'track_count': str(len(tracks)),
                })
        except Exception:
            pass

        return jsonify({"success": True, "playlist_id": playlist_id})
    except Exception as e:
        logger.error(f"Error mirroring playlist: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/mirrored-playlists', methods=['GET'])
def get_mirrored_playlists_endpoint():
    """List all mirrored playlists for the active profile."""
    try:
        database = get_database()
        profile_id = get_current_profile_id()
        playlists = database.get_mirrored_playlists(profile_id=profile_id)
        return jsonify(playlists)
    except Exception as e:
        logger.error(f"Error getting mirrored playlists: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/mirrored-playlists/<int:playlist_id>', methods=['GET'])
def get_mirrored_playlist_endpoint(playlist_id):
    """Get a mirrored playlist with its tracks."""
    try:
        database = get_database()
        playlist = database.get_mirrored_playlist(playlist_id)
        if not playlist:
            return jsonify({"error": "Playlist not found"}), 404
        playlist['tracks'] = database.get_mirrored_playlist_tracks(playlist_id)
        return jsonify(playlist)
    except Exception as e:
        logger.error(f"Error getting mirrored playlist: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/mirrored-playlists/<int:playlist_id>', methods=['DELETE'])
def delete_mirrored_playlist_endpoint(playlist_id):
    """Delete a mirrored playlist."""
    try:
        database = get_database()
        if database.delete_mirrored_playlist(playlist_id):
            return jsonify({"success": True})
        return jsonify({"error": "Playlist not found"}), 404
    except Exception as e:
        logger.error(f"Error deleting mirrored playlist: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/mirrored-playlists/<int:playlist_id>/prepare-discovery', methods=['POST'])
def prepare_mirrored_discovery(playlist_id):
    """Register a mirrored playlist into youtube_playlist_states so the YouTube discovery pipeline can run."""
    try:
        database = get_database()
        playlist = database.get_mirrored_playlist(playlist_id)
        if not playlist:
            return jsonify({"error": "Playlist not found"}), 404

        tracks_data = database.get_mirrored_playlist_tracks(playlist_id)
        url_hash = f"mirrored_{playlist_id}"

        # Build track list in the format the YouTube discovery worker expects
        tracks = [{
            'id': t.get('source_track_id') or f"mirrored_{t['id']}",
            'name': t['track_name'],
            'artists': [t['artist_name']],
            'album': t.get('album_name', ''),
            'duration_ms': t.get('duration_ms', 0)
        } for t in tracks_data]

        playlist_data = {
            'id': url_hash,
            'name': playlist['name'],
            'tracks': tracks,
            'track_count': len(tracks),
            'url': f"mirrored://{playlist['source']}/{playlist['source_playlist_id']}",
            'source': playlist['source']
        }

        youtube_playlist_states[url_hash] = {
            'playlist': playlist_data,
            'phase': 'fresh',
            'discovery_results': [],
            'discovery_progress': 0,
            'spotify_matches': 0,
            'spotify_total': len(tracks),
            'status': 'parsed',
            'url': playlist_data['url'],
            'sync_playlist_id': None,
            'converted_spotify_playlist_id': None,
            'download_process_id': None,
            'created_at': time.time(),
            'last_accessed': time.time(),
            'discovery_future': None,
            'sync_progress': {}
        }

        logger.info(f"Prepared mirrored playlist for discovery: {playlist['name']} ({len(tracks)} tracks)")
        return jsonify({"success": True, "url_hash": url_hash})
    except Exception as e:
        logger.error(f"Error preparing mirrored discovery: {e}")
        return jsonify({"error": str(e)}), 500

@app.route('/api/mirrored-playlists/discovery-states', methods=['GET'])
def get_mirrored_discovery_states():
    """Return discovery states for any mirrored playlists that have active/completed discoveries."""
    try:
        states = []
        for url_hash, state in youtube_playlist_states.items():
            if not url_hash.startswith('mirrored_'):
                continue
            states.append({
                'url_hash': url_hash,
                'playlist_id': int(url_hash.replace('mirrored_', '')),
                'playlist': state['playlist'],
                'phase': state['phase'],
                'status': state.get('status', ''),
                'discovery_progress': state.get('discovery_progress', 0),
                'spotify_matches': state.get('spotify_matches', 0),
                'spotify_total': state.get('spotify_total', 0),
                'discovery_results': state.get('discovery_results', []),
                'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'),
                'download_process_id': state.get('download_process_id'),
            })
        return jsonify({"states": states})
    except Exception as e:
        logger.error(f"Error getting mirrored discovery states: {e}")
        return jsonify({"error": str(e)}), 500

def convert_beatport_results_to_spotify_tracks(discovery_results):
    """Convert Beatport discovery results to Spotify tracks format for sync"""
    spotify_tracks = []

    for result in discovery_results:
        # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery)
        if result.get('spotify_data'):
            spotify_data = result['spotify_data']

            # Convert artists from objects to strings if needed
            artists = spotify_data['artists']
            if isinstance(artists, list) and len(artists) > 0:
                if isinstance(artists[0], dict) and 'name' in artists[0]:
                    # Convert from [{'name': 'Artist'}] to ['Artist']
                    artists = [artist['name'] for artist in artists]

            spotify_tracks.append({
                'id': spotify_data['id'],
                'name': spotify_data['name'],
                'artists': artists,
                'album': spotify_data['album'],
                'source': 'beatport'
            })
        elif result.get('spotify_track') and result.get('status_class') == 'found':
            # Build from individual fields (automatic discovery format)
            spotify_tracks.append({
                'id': result.get('spotify_id', 'unknown'),
                'name': result.get('spotify_track', 'Unknown Track'),
                'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'],
                'album': result.get('spotify_album', 'Unknown Album'),
                'source': 'beatport'
            })

    return spotify_tracks

# Beatport download missing tracks is handled frontend-only (like YouTube)
# No backend endpoint needed - uses existing download modal infrastructure

class WebMetadataUpdateWorker:
    """Web-based metadata update worker - EXACT port of dashboard.py MetadataUpdateWorker"""

    def __init__(self, artists, media_client, spotify_client, server_type, refresh_interval_days=30):
        self.artists = artists
        self.media_client = media_client  # Can be plex_client or jellyfin_client
        self.spotify_client = spotify_client
        self.server_type = server_type  # "plex" or "jellyfin"
        self.matching_engine = MusicMatchingEngine()
        self.refresh_interval_days = refresh_interval_days
        self.should_stop = False
        self.processed_count = 0
        self.successful_count = 0
        self.failed_count = 0
        self.max_workers = 1
        # DB-first: reuse existing metadata from SoulSync database
        try:
            from database.music_database import MusicDatabase
            self._db = MusicDatabase()
        except Exception:
            self._db = None
        self.thread_lock = threading.Lock()
    
    def stop(self):
        self.should_stop = True
    
    def get_artist_name(self, artist):
        """Get artist name consistently across Plex and Jellyfin"""
        return getattr(artist, 'title', 'Unknown Artist')
    
    def run(self):
        """Process all artists one by one - EXACT copy from dashboard.py"""
        global metadata_update_state
        
        try:
            # Load artists in background if not provided - EXACTLY like dashboard.py
            if self.artists is None:
                # Enable lightweight mode for Jellyfin to skip track caching
                if self.server_type == "jellyfin":
                    self.media_client.set_metadata_only_mode(True)
                elif self.server_type == "navidrome":
                    # Navidrome doesn't need special mode setting
                    pass
                
                all_artists = self.media_client.get_all_artists()
                print(f"[DEBUG] Raw artists returned: {[getattr(a, 'title', 'NO_TITLE') for a in (all_artists or [])]}")
                if not all_artists:
                    metadata_update_state['status'] = 'error'
                    metadata_update_state['error'] = f"No artists found in {self.server_type.title()} library"
                    add_activity_item("❌", "Metadata Update", metadata_update_state['error'], "Now")
                    return
                
                # Filter artists that need processing
                artists_to_process = [artist for artist in all_artists if self.artist_needs_processing(artist)]
                self.artists = artists_to_process
                
                # Emit loaded signal equivalent - EXACTLY like dashboard.py
                if len(artists_to_process) == 0:
                    metadata_update_state['status'] = 'completed'
                    metadata_update_state['completed_at'] = datetime.now()
                    add_activity_item("✅", "Metadata Update", "All artists already have good metadata", "Now")
                    return
                else:
                    add_activity_item("🎵", "Metadata Update", f"Processing {len(artists_to_process)} of {len(all_artists)} artists", "Now")
                
                if not artists_to_process:
                    metadata_update_state['status'] = 'completed'
                    metadata_update_state['completed_at'] = datetime.now()
                    return
            
            total_artists = len(self.artists)
            metadata_update_state['total'] = total_artists
            
            # Process artists in parallel using ThreadPoolExecutor - EXACTLY like dashboard.py
            def process_single_artist(artist):
                """Process a single artist and return results"""
                if self.should_stop or metadata_update_state['status'] == 'stopping':
                    return None
                    
                artist_name = getattr(artist, 'title', 'Unknown Artist')
                
                # Double-check ignore flag right before processing
                if self.media_client.is_artist_ignored(artist):
                    return (artist_name, True, "Skipped (ignored)")
                
                try:
                    success, details = self.update_artist_metadata(artist)
                    return (artist_name, success, details)
                except Exception as e:
                    return (artist_name, False, f"Error: {str(e)}")
            
            # Process artists sequentially with rate limiting
            # (no ThreadPoolExecutor — API rate limits make parallelism counterproductive)
            import time
            for artist in self.artists:
                if self.should_stop or metadata_update_state['status'] == 'stopping':
                    break

                result = process_single_artist(artist)
                if result is None:
                    continue

                artist_name, success, details = result

                with self.thread_lock:
                    self.processed_count += 1
                    if success:
                        self.successful_count += 1
                    else:
                        self.failed_count += 1

                progress_percent = (self.processed_count / total_artists) * 100
                metadata_update_state.update({
                    'current_artist': artist_name,
                    'processed': self.processed_count,
                    'percentage': progress_percent,
                    'successful': self.successful_count,
                    'failed': self.failed_count
                })

                # Rate limit: 1.5s between artists (this actually runs between artists now)
                time.sleep(1.5)
            
            # Mark as completed - equivalent to finished.emit
            metadata_update_state['status'] = 'completed'
            metadata_update_state['completed_at'] = datetime.now()
            metadata_update_state['current_artist'] = 'Completed'
            
            summary = f"Processed {self.processed_count} artists: {self.successful_count} updated, {self.failed_count} failed"
            add_activity_item("🎵", "Metadata Complete", summary, "Now")
            
        except Exception as e:
            print(f"Metadata update failed: {e}")
            metadata_update_state['status'] = 'error'
            metadata_update_state['error'] = str(e)
            add_activity_item("❌", "Metadata Error", str(e), "Now")
    
    def artist_needs_processing(self, artist):
        """Check if an artist needs metadata processing using age-based detection - EXACT copy from dashboard.py"""
        try:
            # Check if artist is manually ignored
            if self.media_client.is_artist_ignored(artist):
                return False
            
            # Use media client's age-based checking with configured interval
            return self.media_client.needs_update_by_age(artist, self.refresh_interval_days)
            
        except Exception as e:
            print(f"Error checking artist {getattr(artist, 'title', 'Unknown')}: {e}")
            return True  # Process if we can't determine status
    
    def _check_db_artist(self, artist_name):
        """Check SoulSync DB for existing artist metadata (genres, spotify_artist_id).

        NOTE: DB thumb_url is a Plex/Jellyfin internal path, NOT a downloadable URL.
        Photos must be checked via the media server object, not the DB.

        Returns (db_artist_dict, has_genres, spotify_artist_id) or (None, False, None) if not found."""
        if not self._db:
            return None, False, None
        try:
            db_artists = self._db.search_artists(artist_name, limit=5)
            if not db_artists:
                return None, False, None
            # Find best name match
            best = None
            best_score = 0.0
            norm_name = self.matching_engine.normalize_string(artist_name)
            for dba in db_artists:
                score = self.matching_engine.similarity_score(
                    norm_name, self.matching_engine.normalize_string(dba.name))
                if score > best_score:
                    best_score = score
                    best = dba
            if not best or best_score < 0.85:
                return None, False, None
            has_genres = bool(best.genres and len(best.genres) > 0)
            # Get spotify_artist_id from raw DB row (not in dataclass)
            spotify_artist_id = None
            try:
                raw = self._db.api_get_artist(best.id)
                if raw:
                    spotify_artist_id = raw.get('spotify_artist_id')
            except Exception:
                pass
            return best, has_genres, spotify_artist_id
        except Exception:
            return None, False, None

    def update_artist_metadata(self, artist):
        """Update a single artist's metadata. Checks SoulSync DB first to avoid unnecessary API calls.

        DB-first strategy:
        - Genres: DB stores real genre strings → can apply directly, skip Spotify
        - spotify_artist_id: DB may have it from enrichment → skip search_artists() call
        - Photos/album art: DB thumb_url is a media-server internal path (not downloadable)
          so these MUST come from Spotify API
        """
        try:
            artist_name = getattr(artist, 'title', 'Unknown Artist')

            # Skip processing for artists with no valid name
            if artist_name == 'Unknown Artist' or not artist_name or not artist_name.strip():
                return False, "Skipped: No valid artist name"

            # DB-first: check what we already have cached
            db_artist, db_has_genres, db_spotify_id = self._check_db_artist(artist_name)

            # Check what the media server artist is currently missing
            needs_photo = not self.artist_has_valid_photo(artist) if self.server_type != "jellyfin" else True
            needs_genres = not getattr(artist, 'genres', None)
            needs_album_art = self.server_type == "plex"

            # If media server already has valid photo + genres + album art, skip entirely
            if not needs_photo and not needs_genres and not needs_album_art:
                self.media_client.update_artist_biography(artist)
                return True, "Already up to date"

            # Determine if we actually need Spotify
            # Photos and album art MUST come from Spotify (DB only has internal media server paths)
            # Genres CAN come from DB if available
            need_spotify = needs_photo or needs_album_art or (needs_genres and not db_has_genres)

            spotify_artist = None
            highest_score = 0.0

            if need_spotify:
                # Try direct lookup by cached spotify_artist_id first (1 API call vs search)
                if db_spotify_id:
                    try:
                        from core.spotify_client import Artist as SpotifyArtistDC
                        raw = self.spotify_client.get_artist(db_spotify_id)
                        if raw and 'name' in raw:
                            spotify_artist = SpotifyArtistDC.from_spotify_artist(raw)
                            highest_score = 1.0
                            print(f"✅ Metadata updater: direct Spotify lookup for '{artist_name}' via cached ID {db_spotify_id}")
                    except Exception as e:
                        print(f"Direct Spotify lookup failed for {db_spotify_id}: {e}")
                        spotify_artist = None

                # Fall back to search if direct lookup didn't work
                if not spotify_artist:
                    spotify_artists = self.spotify_client.search_artists(artist_name, limit=5)
                    if not spotify_artists:
                        # Spotify failed — apply DB genres if available, skip photos/art
                        changes_made = []
                        if needs_genres and db_has_genres and db_artist:
                            if self._apply_db_genres(artist, db_artist.genres):
                                changes_made.append("genres (DB)")
                        if changes_made:
                            self.media_client.update_artist_biography(artist)
                            return True, f"Updated {', '.join(changes_made)} (Spotify unavailable)"
                        return False, "Not found on Spotify"

                    # Find the best match
                    best_match = None
                    plex_artist_normalized = self.matching_engine.normalize_string(artist_name)

                    for sa in spotify_artists:
                        spotify_artist_normalized = self.matching_engine.normalize_string(sa.name)
                        score = self.matching_engine.similarity_score(plex_artist_normalized, spotify_artist_normalized)
                        if score > highest_score:
                            highest_score = score
                            best_match = sa

                    if not best_match or highest_score < 0.7:
                        # No good Spotify match — still try DB genres
                        changes_made = []
                        if needs_genres and db_has_genres and db_artist:
                            if self._apply_db_genres(artist, db_artist.genres):
                                changes_made.append("genres (DB)")
                        if changes_made:
                            self.media_client.update_artist_biography(artist)
                            return True, f"Updated {', '.join(changes_made)} (no Spotify match)"
                        return False, f"No confident match found (best: '{getattr(best_match, 'name', 'N/A')}', score: {highest_score:.2f})"

                    spotify_artist = best_match

            changes_made = []

            # Update photo (always from Spotify — DB only has media server paths)
            if needs_photo and spotify_artist:
                photo_updated = self.update_artist_photo(artist, spotify_artist)
                if photo_updated:
                    changes_made.append("photo")

            # Update genres — use DB if available, otherwise Spotify
            if needs_genres:
                if db_has_genres and db_artist:
                    genres_updated = self._apply_db_genres(artist, db_artist.genres)
                    if genres_updated:
                        changes_made.append("genres (DB)")
                    elif spotify_artist:
                        # DB genres didn't result in changes, try Spotify for newer/different genres
                        genres_updated = self.update_artist_genres(artist, spotify_artist)
                        if genres_updated:
                            changes_made.append("genres")
                elif spotify_artist:
                    genres_updated = self.update_artist_genres(artist, spotify_artist)
                    if genres_updated:
                        changes_made.append("genres")

            # Update album artwork (only for Plex, always from Spotify)
            if self.server_type == "plex" and spotify_artist:
                albums_updated = self.update_album_artwork(artist, spotify_artist)
                if albums_updated > 0:
                    changes_made.append(f"{albums_updated} album art")
            elif self.server_type != "plex":
                print(f"Skipping album artwork updates for Jellyfin artist: {artist.title}")

            if changes_made:
                biography_updated = self.media_client.update_artist_biography(artist)
                if biography_updated:
                    changes_made.append("timestamp")

                source = f"match: '{spotify_artist.name}', score: {highest_score:.2f}" if spotify_artist else "DB cache"
                details = f"Updated {', '.join(changes_made)} ({source})"
                return True, details
            else:
                self.media_client.update_artist_biography(artist)
                return True, "Already up to date"

        except Exception as e:
            return False, str(e)

    def _apply_db_genres(self, artist, genres):
        """Apply genres from DB cache to media server."""
        try:
            if not genres:
                return False
            existing_genres = set(genre.tag if hasattr(genre, 'tag') else str(genre)
                                for genre in (getattr(artist, 'genres', None) or []))
            db_genres = set(g for g in genres if g and g.strip() and len(g.strip()) > 1)
            if db_genres and db_genres != existing_genres:
                return self.media_client.update_artist_genres(artist, list(db_genres)[:10])
            return False
        except Exception:
            return False
    
    def update_artist_photo(self, artist, spotify_artist):
        """Update artist photo from Spotify - EXACT copy from dashboard.py"""
        try:
            # Check if artist already has a good photo (skip check for Jellyfin)
            if self.server_type != "jellyfin" and self.artist_has_valid_photo(artist):
                print(f"🖼️ Skipping {artist.title}: already has valid photo ({getattr(artist, 'thumb', 'None')})")
                return False

            # Get the image URL from Spotify
            if not spotify_artist.image_url:
                print(f"🚫 Skipping {artist.title}: no Spotify image URL available")
                return False

            print(f"📸 Processing {artist.title}: downloading from Spotify...")
                
            image_url = spotify_artist.image_url
            
            # Download and validate image
            response = requests.get(image_url, timeout=10)
            response.raise_for_status()

            # Validate and convert image (skip conversion for Jellyfin to preserve format)
            if self.server_type == "jellyfin":
                # For Jellyfin, use raw image data to preserve original format
                image_data = response.content
                print(f"📸 Using raw image data for Jellyfin ({len(image_data)} bytes)")
            else:
                # For other servers, validate and convert
                image_data = self.validate_and_convert_image(response.content)
                if not image_data:
                    return False
            
            # Upload to media server using client's method
            return self.media_client.update_artist_poster(artist, image_data)
            
        except Exception as e:
            print(f"Error updating photo for {getattr(artist, 'title', 'Unknown')}: {e}")
            return False
    
    def update_artist_genres(self, artist, spotify_artist):
        """Update artist genres from Spotify and albums - EXACT copy from dashboard.py"""
        try:
            # Get existing genres
            existing_genres = set(genre.tag if hasattr(genre, 'tag') else str(genre) 
                                for genre in (artist.genres or []))
            
            # Get Spotify artist genres
            spotify_genres = set(spotify_artist.genres or [])
            
            # Get genres from all albums
            album_genres = set()
            try:
                for album in artist.albums():
                    if hasattr(album, 'genres') and album.genres:
                        album_genres.update(genre.tag if hasattr(genre, 'tag') else str(genre) 
                                          for genre in album.genres)
            except Exception:
                pass  # Albums might not be accessible
            
            # Combine all genres (prioritize Spotify genres)
            all_genres = spotify_genres.union(album_genres)
            
            # Filter out empty/invalid genres
            all_genres = {g for g in all_genres if g and g.strip() and len(g.strip()) > 1}
            
            # Only update if we have new genres and they're different
            if all_genres and (not existing_genres or all_genres != existing_genres):
                # Convert to list and limit to 10 genres
                genre_list = list(all_genres)[:10]
                
                # Use media client API to update genres
                success = self.media_client.update_artist_genres(artist, genre_list)
                if success:
                    return True
                else:
                    return False
            else:
                return False
            
        except Exception as e:
            print(f"Error updating genres for {getattr(artist, 'title', 'Unknown')}: {e}")
            return False
    
    def update_album_artwork(self, artist, spotify_artist):
        """Update album artwork for all albums by this artist from Spotify.
        DB thumb_url is a media-server internal path, so album art must come from Spotify."""
        try:
            updated_count = 0
            skipped_count = 0

            # Get all albums for this artist
            try:
                albums = list(artist.albums())
            except Exception:
                print(f"Could not access albums for artist '{artist.title}'")
                return 0

            if not albums:
                print(f"No albums found for artist '{artist.title}'")
                return 0

            import time
            for album in albums:
                try:
                    album_title = getattr(album, 'title', 'Unknown Album')

                    # Check if album already has good artwork on the media server
                    if self.album_has_valid_artwork(album):
                        skipped_count += 1
                        continue

                    # Rate limit between album API calls
                    time.sleep(0.5)

                    # Search for this specific album on Spotify
                    album_query = f"album:{album_title} artist:{spotify_artist.name}"
                    spotify_albums = self.spotify_client.search_albums(album_query, limit=3)

                    if not spotify_albums:
                        continue

                    # Find the best matching album
                    best_album = None
                    highest_score = 0.0

                    plex_album_normalized = self.matching_engine.normalize_string(album_title)

                    for spotify_album in spotify_albums:
                        spotify_album_normalized = self.matching_engine.normalize_string(spotify_album.name)
                        score = self.matching_engine.similarity_score(plex_album_normalized, spotify_album_normalized)

                        if score > highest_score:
                            highest_score = score
                            best_album = spotify_album

                    # If we found a good match with artwork, download it
                    if best_album and highest_score > 0.7 and best_album.image_url:
                        if self.download_and_upload_album_artwork(album, best_album.image_url):
                            updated_count += 1

                except Exception as e:
                    print(f"Error processing album '{getattr(album, 'title', 'Unknown')}': {e}")
                    continue

            return updated_count

        except Exception as e:
            print(f"Error updating album artwork for artist '{getattr(artist, 'title', 'Unknown')}': {e}")
            return 0
    
    def album_has_valid_artwork(self, album):
        """Check if album has valid artwork - EXACT copy from dashboard.py"""
        try:
            if not hasattr(album, 'thumb') or not album.thumb:
                return False
            
            thumb_url = str(album.thumb)
            
            # Completely empty or None
            if not thumb_url or thumb_url.strip() == '':
                return False
            
            # Obvious placeholder text in URL
            obvious_placeholders = ['no-image', 'placeholder', 'missing', 'default-album', 'blank.jpg', 'empty.png']
            thumb_lower = thumb_url.lower()
            for placeholder in obvious_placeholders:
                if placeholder in thumb_lower:
                    return False
            
            # Extremely short URLs (likely broken)
            if len(thumb_url) < 20:
                return False
            
            return True
            
        except Exception as e:
            return True
    
    def download_and_upload_album_artwork(self, album, image_url):
        """Download artwork from Spotify and upload to media server - EXACT copy from dashboard.py"""
        try:
            # Download image from Spotify
            response = requests.get(image_url, timeout=10)
            response.raise_for_status()
            
            # Validate and convert image
            image_data = self.validate_and_convert_image(response.content)
            if not image_data:
                return False
            
            # Upload using media client
            success = self.media_client.update_album_poster(album, image_data)
            return success
            
        except Exception as e:
            print(f"Error downloading/uploading artwork for album '{getattr(album, 'title', 'Unknown')}': {e}")
            return False
    
    def artist_has_valid_photo(self, artist):
        """Check if artist has a valid photo - EXACT copy from dashboard.py"""
        try:
            if not hasattr(artist, 'thumb') or not artist.thumb:
                return False
            
            thumb_url = str(artist.thumb)
            if 'default' in thumb_url.lower() or len(thumb_url) < 50:
                return False
            
            return True
            
        except Exception:
            return False
    
    def validate_and_convert_image(self, image_data):
        """Validate and convert image for media server compatibility - EXACT copy from dashboard.py"""
        try:
            from PIL import Image
            import io
            
            # Open and validate image
            image = Image.open(io.BytesIO(image_data))
            
            # Check minimum dimensions
            width, height = image.size
            if width < 200 or height < 200:
                return None
            
            # Convert to JPEG for consistency
            if image.format != 'JPEG':
                buffer = io.BytesIO()
                image.convert('RGB').save(buffer, format='JPEG', quality=95)
                return buffer.getvalue()
            
            return image_data
            
        except Exception:
            return None
    
    def upload_artist_poster(self, artist, image_data):
        """Upload poster using media client - EXACT copy from dashboard.py"""
        try:
            # Use media client's update method if available
            if hasattr(self.media_client, 'update_artist_poster'):
                return self.media_client.update_artist_poster(artist, image_data)
            
            # Fallback for Plex: direct API call
            if self.server_type == "plex":
                import requests
                server = self.media_client.server
                upload_url = f"{server._baseurl}/library/metadata/{artist.ratingKey}/posters"
                headers = {
                    'X-Plex-Token': server._token,
                    'Content-Type': 'image/jpeg'
                }
                
                response = requests.post(upload_url, data=image_data, headers=headers)
                response.raise_for_status()
                
                # Refresh artist to see changes
                artist.refresh()
                return True

            # Jellyfin: Use Jellyfin API to upload artist image
            elif self.server_type == "jellyfin":
                import requests
                jellyfin_config = config_manager.get_jellyfin_config()
                jellyfin_base_url = jellyfin_config.get('base_url', '')
                jellyfin_token = jellyfin_config.get('api_key', '')

                if not jellyfin_base_url or not jellyfin_token:
                    print("❌ Jellyfin configuration missing for image upload")
                    return False

                upload_url = f"{jellyfin_base_url.rstrip('/')}/Items/{artist.ratingKey}/Images/Primary"
                headers = {
                    'Authorization': f'MediaBrowser Token="{jellyfin_token}"',
                    'Content-Type': 'image/jpeg'
                }

                response = requests.post(upload_url, data=image_data, headers=headers)
                response.raise_for_status()
                return True

            # Navidrome: Currently not supported (Subsonic API doesn't support image uploads)
            elif self.server_type == "navidrome":
                print("ℹ️ Navidrome does not support artist image uploads via Subsonic API")
                return False

            else:
                # Unknown server type
                return False
            
        except Exception as e:
            print(f"Error uploading poster: {e}")
            return False

# --- Docker Helper Functions ---

def docker_resolve_url(url):
    """
    Resolve localhost URLs to Docker host when running in container
    """
    import os
    if os.path.exists('/.dockerenv') and 'localhost' in url:
        return url.replace('localhost', 'host.docker.internal')
    return url

# --- Main Execution ---

def start_oauth_callback_servers():
    """Start dedicated OAuth callback servers for Spotify and Tidal"""
    import threading
    from http.server import HTTPServer, BaseHTTPRequestHandler
    import urllib.parse
    
    # Spotify callback server (port 8888 — for direct/local access only)
    class SpotifyCallbackHandler(BaseHTTPRequestHandler):
        def do_GET(self):
            parsed_url = urllib.parse.urlparse(self.path)

            # Only process requests to /callback — ignore everything else
            if parsed_url.path != '/callback':
                self.send_response(404)
                self.send_header('Content-type', 'text/plain')
                self.end_headers()
                self.wfile.write(b'Not found. Spotify callback is at /callback')
                return

            query_params = urllib.parse.parse_qs(parsed_url.query)
            print(f"🎵 Spotify callback received on port 8888: {self.path}")

            if 'code' in query_params:
                auth_code = query_params['code'][0]
                print(f"🎵 Received Spotify authorization code: {auth_code[:10]}...")

                # Manually trigger the token exchange using spotipy's auth manager
                try:
                    from core.spotify_client import SpotifyClient
                    from spotipy.oauth2 import SpotifyOAuth
                    from config.settings import config_manager

                    # Get Spotify config
                    config = config_manager.get_spotify_config()
                    configured_uri = config.get('redirect_uri', "http://127.0.0.1:8888/callback")
                    print(f"🎵 Using redirect_uri for token exchange: {configured_uri}")

                    # Create auth manager and exchange code for token
                    auth_manager = SpotifyOAuth(
                        client_id=config['client_id'],
                        client_secret=config['client_secret'],
                        redirect_uri=configured_uri,
                        scope="user-library-read user-read-private playlist-read-private playlist-read-collaborative user-read-email",
                        cache_path='config/.spotify_cache'
                    )

                    # Extract the authorization code and exchange it for tokens
                    token_info = auth_manager.get_access_token(auth_code, as_dict=True)

                    if token_info:
                        # Reinitialize the global client with new tokens
                        global spotify_client
                        spotify_client = SpotifyClient()

                        if spotify_client.is_authenticated():
                            # Invalidate status cache so next poll picks up the new connection
                            _status_cache_timestamps['spotify'] = 0
                            # Refresh enrichment worker's client so it picks up new auth
                            if spotify_enrichment_worker and hasattr(spotify_enrichment_worker, 'client'):
                                spotify_enrichment_worker.client.reload_config()
                            add_activity_item("✅", "Spotify Auth Complete", "Successfully authenticated with Spotify", "Now")
                            self.send_response(200)
                            self.send_header('Content-type', 'text/html')
                            self.end_headers()
                            self.wfile.write(b'<h1>Spotify Authentication Successful!</h1><p>You can close this window.</p>')
                        else:
                            raise Exception("Token exchange succeeded but authentication validation failed")
                    else:
                        raise Exception("Failed to exchange authorization code for access token")
                except Exception as e:
                    print(f"🔴 Spotify token processing error: {e}")
                    add_activity_item("❌", "Spotify Auth Failed", f"Token processing failed: {str(e)}", "Now")
                    self.send_response(400)
                    self.send_header('Content-type', 'text/html')
                    self.end_headers()
                    self.wfile.write(f'<h1>Spotify Authentication Failed</h1><p>{str(e)}</p>'.encode())
            elif 'error' in query_params:
                error = query_params['error'][0]
                print(f"🔴 Spotify OAuth error returned by Spotify: {error}")
                print(f"🔴 Full callback URL: {self.path}")
                add_activity_item("❌", "Spotify Auth Failed", f"Spotify returned error: {error}", "Now")
                self.send_response(400)
                self.send_header('Content-type', 'text/html')
                self.end_headers()
                self.wfile.write(f'<h1>Spotify Authentication Failed</h1><p>Spotify returned error: {error}</p>'.encode())
            else:
                # No code AND no error — callback was hit without OAuth params
                print(f"🔴 Spotify callback received without OAuth parameters (no code or error)")
                print(f"🔴 Path: {self.path} | Query params: {query_params}")
                print(f"🔴 This usually means the redirect lost its query parameters (reverse proxy issue)")
                self.send_response(400)
                self.send_header('Content-type', 'text/html')
                self.end_headers()
                msg = (
                    '<h1>Spotify Authentication Failed</h1>'
                    '<p>The callback was received but no authorization code or error was included.</p>'
                    '<p><strong>If you are using a reverse proxy:</strong> Your proxy may be stripping query parameters '
                    'during the redirect. Try setting your Spotify redirect URI to use port 8008 instead '
                    '(e.g. <code>https://yourdomain.com/callback</code>) — the main app handles callbacks too.</p>'
                )
                self.wfile.write(msg.encode())
        
        def log_message(self, format, *args):
            pass  # Suppress server logs
    
    # Start Spotify callback server
    def run_spotify_server():
        try:
            spotify_server = HTTPServer(('0.0.0.0', 8888), SpotifyCallbackHandler)
            print("🎵 Started Spotify OAuth callback server on port 8888")
            spotify_server.serve_forever()
        except Exception as e:
            print(f"🔴 Failed to start Spotify callback server: {e}")
    
    # Tidal callback server  
    class TidalCallbackHandler(BaseHTTPRequestHandler):
        def do_GET(self):
            print("🎶🎶🎶 TIDAL CALLBACK SERVER RECEIVED REQUEST 🎶🎶🎶")
            parsed_url = urllib.parse.urlparse(self.path)
            query_params = urllib.parse.parse_qs(parsed_url.query)
            print(f"🎶 Callback path: {self.path}")
            
            if 'code' in query_params:
                auth_code = query_params['code'][0]
                print(f"🎶 Received Tidal authorization code: {auth_code[:10]}...")
                
                # Exchange the authorization code for tokens
                try:
                    from core.tidal_client import TidalClient
                    
                    # Create a temporary client and set the stored PKCE values
                    temp_client = TidalClient()
                    
                    # Restore the PKCE values from the auth request
                    global tidal_oauth_state
                    with tidal_oauth_lock:
                        temp_client.code_verifier = tidal_oauth_state["code_verifier"]
                        temp_client.code_challenge = tidal_oauth_state["code_challenge"]
                    
                    print(f"🔐 Restored PKCE - verifier: {temp_client.code_verifier[:20] if temp_client.code_verifier else 'None'}... challenge: {temp_client.code_challenge[:20] if temp_client.code_challenge else 'None'}...")
                    
                    success = temp_client.fetch_token_from_code(auth_code)
                    
                    if success:
                        # Reinitialize the global tidal client with new tokens
                        global tidal_client
                        tidal_client = TidalClient()
                        
                        add_activity_item("✅", "Tidal Auth Complete", "Successfully authenticated with Tidal", "Now")
                        self.send_response(200)
                        self.send_header('Content-type', 'text/html')
                        self.end_headers()
                        self.wfile.write(b'<h1>Tidal Authentication Successful!</h1><p>You can close this window.</p>')
                    else:
                        raise Exception("Failed to exchange authorization code for tokens")
                        
                except Exception as e:
                    print(f"🔴 Tidal token processing error: {e}")
                    add_activity_item("❌", "Tidal Auth Failed", f"Token processing failed: {str(e)}", "Now")
                    self.send_response(400)
                    self.send_header('Content-type', 'text/html')
                    self.end_headers()
                    self.wfile.write(f'<h1>Tidal Authentication Failed</h1><p>{str(e)}</p>'.encode())
            else:
                error = query_params.get('error', ['Unknown error'])[0]
                print(f"🔴 Tidal OAuth error: {error}")
                add_activity_item("❌", "Tidal Auth Failed", f"OAuth error: {error}", "Now")
                self.send_response(400)
                self.send_header('Content-type', 'text/html')
                self.end_headers()
                self.wfile.write(f'<h1>Tidal Authentication Failed</h1><p>{error}</p>'.encode())
        
        def log_message(self, format, *args):
            pass  # Suppress server logs
    
    def run_tidal_server():
        try:
            tidal_server = HTTPServer(('0.0.0.0', 8889), TidalCallbackHandler)
            print("🎶 Started Tidal OAuth callback server on port 8889")
            print(f"🎶 Tidal server listening on all interfaces, port 8889")
            tidal_server.serve_forever()
        except Exception as e:
            print(f"🔴 Failed to start Tidal callback server: {e}")
            import traceback
            print(f"🔴 Full error: {traceback.format_exc()}")
    
    # Start both servers in background threads
    spotify_thread = threading.Thread(target=run_spotify_server, daemon=True)
    tidal_thread = threading.Thread(target=run_tidal_server, daemon=True)
    
    spotify_thread.start()
    tidal_thread.start()
    
    print("✅ OAuth callback servers started")

# ===============================================
# Artist Detail Spotify Integration Functions
# ===============================================

def get_spotify_artist_discography(artist_name):
    """Get complete artist discography from Spotify using proper matching"""
    try:
        from core.spotify_client import SpotifyClient
        from core.matching_engine import MusicMatchingEngine

        print(f"🎵 Searching Spotify for artist: {artist_name}")

        # Initialize clients
        spotify_client = SpotifyClient()
        matching_engine = MusicMatchingEngine()

        # Search for multiple potential matches (not just 1)
        artists = spotify_client.search_artists(artist_name, limit=5)

        if not artists:
            return {
                'success': False,
                'error': f'Artist "{artist_name}" not found on Spotify'
            }

        # Since database names are exact Spotify names, try exact match first
        best_match = None
        highest_score = 0.0

        # Step 1: Try exact case-insensitive match
        for spotify_artist in artists:
            if artist_name.lower().strip() == spotify_artist.name.lower().strip():
                print(f"🎯 Exact match found: '{spotify_artist.name}'")
                best_match = spotify_artist
                highest_score = 1.0
                break

        # Step 2: If no exact match, use matching engine with higher threshold
        if not best_match:
            db_artist_normalized = matching_engine.normalize_string(artist_name)

            for spotify_artist in artists:
                spotify_artist_normalized = matching_engine.normalize_string(spotify_artist.name)
                score = matching_engine.similarity_score(db_artist_normalized, spotify_artist_normalized)

                print(f"🔍 Fuzzy match candidate: '{spotify_artist.name}' (score: {score:.3f})")

                if score > highest_score:
                    highest_score = score
                    best_match = spotify_artist

        # Require high confidence threshold since DB should have exact names
        if not best_match or highest_score < 0.95:
            return {
                'success': False,
                'error': f'No confident artist match found for "{artist_name}" (best: "{getattr(best_match, "name", "N/A")}", score: {highest_score:.3f})'
            }

        artist = best_match
        spotify_artist_id = artist.id

        print(f"🎵 Found Spotify artist: {artist.name} (ID: {spotify_artist_id}, confidence: {highest_score:.3f})")

        # Get all albums (albums, singles, and compilations)
        all_albums = spotify_client.get_artist_albums(spotify_artist_id, album_type='album,single,compilation', limit=50)

        if not all_albums:
            return {
                'success': False,
                'error': f'No albums found for artist "{artist_name}"'
            }

        print(f"📀 Found {len(all_albums)} releases on Spotify")

        # Categorize releases
        albums = []
        eps = []
        singles = []

        for album in all_albums:
            # Skip albums where this artist isn't the primary (first-listed) artist
            if getattr(album, 'artist_ids', None) and album.artist_ids[0] != spotify_artist_id:
                continue

            # Use the Album object properties
            track_count = album.total_tracks

            release_data = {
                'title': album.name,
                'year': album.release_date[:4] if album.release_date else None,
                'release_date': album.release_date if album.release_date else None,
                'image_url': album.image_url,
                'spotify_id': album.id,
                'owned': False,  # Will be updated when merging with owned data
                'track_count': track_count,
                'album_type': album.album_type.lower()
            }

            # Categorize based on album type and track count
            album_type = album.album_type.lower()

            if album_type == 'single' or track_count <= 3:
                singles.append(release_data)
            elif album_type == 'ep' or (track_count >= 4 and track_count <= 6):
                eps.append(release_data)
            elif album_type == 'compilation':
                # Compilations go with albums
                albums.append(release_data)
            else:
                albums.append(release_data)

        print(f"📀 Categorized Spotify releases - Albums: {len(albums)}, EPs: {len(eps)}, Singles: {len(singles)}")

        return {
            'success': True,
            'albums': albums,
            'eps': eps,
            'singles': singles,
            'artist_image': artist.image_url if hasattr(artist, 'image_url') else None,
            'spotify_artist_id': spotify_artist_id,
            'spotify_artist_name': artist.name
        }

    except Exception as e:
        print(f"❌ Error getting Spotify discography for {artist_name}: {e}")
        return {
            'success': False,
            'error': str(e)
        }

def merge_discography_data(owned_releases, spotify_discography, db=None, artist_name=None):
    """Build discography from Spotify data with 'checking' state - ownership is resolved via SSE stream"""
    try:
        print("🔄 Building discography cards (fast path - no DB matching)...")

        def build_category(spotify_category, category_name):
            """Build cards for a category with checking state"""
            cards = []
            for spotify_release in spotify_category:
                card = {
                    'title': spotify_release['title'],
                    'spotify_id': spotify_release.get('spotify_id'),
                    'album_type': spotify_release.get('album_type', 'album'),
                    'image_url': spotify_release.get('image_url'),
                    'year': spotify_release.get('year'),
                    'track_count': spotify_release.get('track_count', 0),
                    'owned': None,  # null = checking (resolved by completion stream)
                    'track_completion': 'checking',
                }
                if spotify_release.get('release_date'):
                    card['release_date'] = spotify_release['release_date']
                elif spotify_release.get('year'):
                    card['release_date'] = f"{spotify_release['year']}-01-01"
                cards.append(card)
            return cards

        albums = build_category(spotify_discography['albums'], 'Albums')
        eps = build_category(spotify_discography['eps'], 'EPs')
        singles = build_category(spotify_discography['singles'], 'Singles')

        print(f"✅ Built discography cards - Albums: {len(albums)}, EPs: {len(eps)}, Singles: {len(singles)}")

        return {
            'success': True,
            'albums': albums,
            'eps': eps,
            'singles': singles
        }

    except Exception as e:
        print(f"❌ Error building discography: {e}")
        import traceback
        traceback.print_exc()
        return {
            'success': False,
            'error': str(e),
            'albums': [],
            'eps': [],
            'singles': []
        }

# ================================================================================================
# MUSICBRAINZ ENRICHMENT - PHASE 5 WEB UI INTEGRATION
# ================================================================================================

# --- MusicBrainz Worker Initialization ---
mb_worker = None
try:
    from database.music_database import MusicDatabase
    mb_db = MusicDatabase()
    mb_worker = MusicBrainzWorker(
        database=mb_db,
        app_name="SoulSync",
        app_version="1.0",
        contact_email=""
    )
    # Start worker automatically (can be paused via UI)
    mb_worker.start()
    print("✅ MusicBrainz enrichment worker initialized and started")
except Exception as e:
    print(f"⚠️ MusicBrainz worker initialization failed: {e}")
    mb_worker = None

# --- MusicBrainz API Endpoints ---

@app.route('/api/musicbrainz/status', methods=['GET'])
def musicbrainz_status():
    """Get MusicBrainz enrichment status for UI polling"""
    try:
        if mb_worker is None:
            return jsonify({
                'enabled': False,
                'running': False,
                'paused': False,
                'current_item': None,
                'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0},
                'progress': {}
            }), 200
        
        status = mb_worker.get_stats()
        return jsonify(status), 200
    except Exception as e:
        logger.error(f"Error getting MusicBrainz status: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/musicbrainz/pause', methods=['POST'])
def musicbrainz_pause():
    """Pause MusicBrainz enrichment worker (finishes current match first)"""
    try:
        if mb_worker is None:
            return jsonify({'error': 'MusicBrainz worker not initialized'}), 400
        
        mb_worker.pause()
        logger.info("MusicBrainz worker paused via UI")
        return jsonify({'status': 'paused'}), 200
    except Exception as e:
        logger.error(f"Error pausing MusicBrainz worker: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/musicbrainz/resume', methods=['POST'])
def musicbrainz_resume():
    """Resume MusicBrainz enrichment worker"""
    try:
        if mb_worker is None:
            return jsonify({'error': 'MusicBrainz worker not initialized'}), 400
        
        mb_worker.resume()
        logger.info("MusicBrainz worker resumed via UI")
        return jsonify({'status': 'running'}), 200
    except Exception as e:
        logger.error(f"Error resuming MusicBrainz worker: {e}")
        return jsonify({'error': str(e)}), 500

# ================================================================================================
# END MUSICBRAINZ INTEGRATION
# ================================================================================================


# ================================================================================================
# AUDIODB ENRICHMENT - ARTIST METADATA & IMAGES
# ================================================================================================

# --- AudioDB Worker Initialization ---
audiodb_worker = None
try:
    from database.music_database import MusicDatabase
    audiodb_db = MusicDatabase()
    audiodb_worker = AudioDBWorker(database=audiodb_db)
    audiodb_worker.start()
    print("✅ AudioDB enrichment worker initialized and started")
except Exception as e:
    print(f"⚠️ AudioDB worker initialization failed: {e}")
    audiodb_worker = None

# --- AudioDB API Endpoints ---

@app.route('/api/audiodb/status', methods=['GET'])
def audiodb_status():
    """Get AudioDB enrichment status for UI polling"""
    try:
        if audiodb_worker is None:
            return jsonify({
                'enabled': False,
                'running': False,
                'paused': False,
                'current_item': None,
                'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0},
                'progress': {}
            }), 200

        status = audiodb_worker.get_stats()
        return jsonify(status), 200
    except Exception as e:
        logger.error(f"Error getting AudioDB status: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/audiodb/pause', methods=['POST'])
def audiodb_pause():
    """Pause AudioDB enrichment worker"""
    try:
        if audiodb_worker is None:
            return jsonify({'error': 'AudioDB worker not initialized'}), 400

        audiodb_worker.pause()
        logger.info("AudioDB worker paused via UI")
        return jsonify({'status': 'paused'}), 200
    except Exception as e:
        logger.error(f"Error pausing AudioDB worker: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/audiodb/resume', methods=['POST'])
def audiodb_resume():
    """Resume AudioDB enrichment worker"""
    try:
        if audiodb_worker is None:
            return jsonify({'error': 'AudioDB worker not initialized'}), 400

        audiodb_worker.resume()
        logger.info("AudioDB worker resumed via UI")
        return jsonify({'status': 'running'}), 200
    except Exception as e:
        logger.error(f"Error resuming AudioDB worker: {e}")
        return jsonify({'error': str(e)}), 500

# ================================================================================================
# END AUDIODB INTEGRATION
# ================================================================================================


# ================================================================================================
# DEEZER ENRICHMENT INTEGRATION
# ================================================================================================

# --- Deezer Worker Initialization ---
deezer_worker = None
try:
    from database.music_database import MusicDatabase
    deezer_db = MusicDatabase()
    deezer_worker = DeezerWorker(database=deezer_db)
    deezer_worker.start()
    print("✅ Deezer enrichment worker initialized and started")
except Exception as e:
    print(f"⚠️ Deezer worker initialization failed: {e}")
    deezer_worker = None

# --- Deezer API Endpoints ---

@app.route('/api/deezer/status', methods=['GET'])
def deezer_status():
    """Get Deezer enrichment status for UI polling"""
    try:
        if deezer_worker is None:
            return jsonify({
                'enabled': False,
                'running': False,
                'paused': False,
                'current_item': None,
                'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0},
                'progress': {}
            }), 200

        status = deezer_worker.get_stats()
        return jsonify(status), 200
    except Exception as e:
        logger.error(f"Error getting Deezer status: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/deezer/pause', methods=['POST'])
def deezer_pause():
    """Pause Deezer enrichment worker"""
    try:
        if deezer_worker is None:
            return jsonify({'error': 'Deezer worker not initialized'}), 400

        deezer_worker.pause()
        logger.info("Deezer worker paused via UI")
        return jsonify({'status': 'paused'}), 200
    except Exception as e:
        logger.error(f"Error pausing Deezer worker: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/deezer/resume', methods=['POST'])
def deezer_resume():
    """Resume Deezer enrichment worker"""
    try:
        if deezer_worker is None:
            return jsonify({'error': 'Deezer worker not initialized'}), 400

        deezer_worker.resume()
        logger.info("Deezer worker resumed via UI")
        return jsonify({'status': 'running'}), 200
    except Exception as e:
        logger.error(f"Error resuming Deezer worker: {e}")
        return jsonify({'error': str(e)}), 500

# ================================================================================================
# END DEEZER INTEGRATION
# ================================================================================================


# ================================================================================================
# SPOTIFY ENRICHMENT INTEGRATION
# ================================================================================================

# --- Spotify Worker Initialization ---
spotify_enrichment_worker = None
try:
    from database.music_database import MusicDatabase
    spotify_enrichment_db = MusicDatabase()
    spotify_enrichment_worker = SpotifyWorker(database=spotify_enrichment_db)
    spotify_enrichment_worker.start()
    print("✅ Spotify enrichment worker initialized and started")
except Exception as e:
    print(f"⚠️ Spotify enrichment worker initialization failed: {e}")
    spotify_enrichment_worker = None

# --- Spotify API Endpoints ---

@app.route('/api/spotify-enrichment/status', methods=['GET'])
def spotify_enrichment_status():
    """Get Spotify enrichment status for UI polling"""
    try:
        if spotify_enrichment_worker is None:
            return jsonify({
                'enabled': False,
                'running': False,
                'paused': False,
                'current_item': None,
                'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0},
                'progress': {}
            }), 200

        status = spotify_enrichment_worker.get_stats()
        return jsonify(status), 200
    except Exception as e:
        logger.error(f"Error getting Spotify enrichment status: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/spotify-enrichment/pause', methods=['POST'])
def spotify_enrichment_pause():
    """Pause Spotify enrichment worker"""
    try:
        if spotify_enrichment_worker is None:
            return jsonify({'error': 'Spotify enrichment worker not initialized'}), 400

        spotify_enrichment_worker.pause()
        logger.info("Spotify enrichment worker paused via UI")
        return jsonify({'status': 'paused'}), 200
    except Exception as e:
        logger.error(f"Error pausing Spotify enrichment worker: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/spotify-enrichment/resume', methods=['POST'])
def spotify_enrichment_resume():
    """Resume Spotify enrichment worker"""
    try:
        if spotify_enrichment_worker is None:
            return jsonify({'error': 'Spotify enrichment worker not initialized'}), 400

        spotify_enrichment_worker.resume()
        logger.info("Spotify enrichment worker resumed via UI")
        return jsonify({'status': 'running'}), 200
    except Exception as e:
        logger.error(f"Error resuming Spotify enrichment worker: {e}")
        return jsonify({'error': str(e)}), 500

# ================================================================================================
# END SPOTIFY ENRICHMENT INTEGRATION
# ================================================================================================


# ================================================================================================
# ITUNES ENRICHMENT INTEGRATION
# ================================================================================================

# --- iTunes Worker Initialization ---
itunes_enrichment_worker = None
try:
    from database.music_database import MusicDatabase
    itunes_enrichment_db = MusicDatabase()
    itunes_enrichment_worker = iTunesWorker(database=itunes_enrichment_db)
    itunes_enrichment_worker.start()
    print("✅ iTunes enrichment worker initialized and started")
except Exception as e:
    print(f"⚠️ iTunes enrichment worker initialization failed: {e}")
    itunes_enrichment_worker = None

# --- iTunes API Endpoints ---

@app.route('/api/itunes-enrichment/status', methods=['GET'])
def itunes_enrichment_status():
    """Get iTunes enrichment status for UI polling"""
    try:
        if itunes_enrichment_worker is None:
            return jsonify({
                'enabled': False,
                'running': False,
                'paused': False,
                'current_item': None,
                'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0},
                'progress': {}
            }), 200

        status = itunes_enrichment_worker.get_stats()
        return jsonify(status), 200
    except Exception as e:
        logger.error(f"Error getting iTunes enrichment status: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/itunes-enrichment/pause', methods=['POST'])
def itunes_enrichment_pause():
    """Pause iTunes enrichment worker"""
    try:
        if itunes_enrichment_worker is None:
            return jsonify({'error': 'iTunes enrichment worker not initialized'}), 400

        itunes_enrichment_worker.pause()
        logger.info("iTunes enrichment worker paused via UI")
        return jsonify({'status': 'paused'}), 200
    except Exception as e:
        logger.error(f"Error pausing iTunes enrichment worker: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/itunes-enrichment/resume', methods=['POST'])
def itunes_enrichment_resume():
    """Resume iTunes enrichment worker"""
    try:
        if itunes_enrichment_worker is None:
            return jsonify({'error': 'iTunes enrichment worker not initialized'}), 400

        itunes_enrichment_worker.resume()
        logger.info("iTunes enrichment worker resumed via UI")
        return jsonify({'status': 'running'}), 200
    except Exception as e:
        logger.error(f"Error resuming iTunes enrichment worker: {e}")
        return jsonify({'error': str(e)}), 500

# ================================================================================================
# END ITUNES ENRICHMENT INTEGRATION
# ================================================================================================


# ================================================================================================
# HYDRABASE P2P MIRROR WORKER
# ================================================================================================

# --- Hydrabase Worker & Client Initialization ---
hydrabase_worker = None
hydrabase_client = None
try:
    def _get_hydrabase_ws_and_lock():
        return (_hydrabase_ws, _hydrabase_lock)
    hydrabase_worker = HydrabaseWorker(get_ws_and_lock=_get_hydrabase_ws_and_lock)
    hydrabase_worker.start()
    hydrabase_client = HydrabaseClient(get_ws_and_lock=_get_hydrabase_ws_and_lock)
    print("✅ Hydrabase P2P mirror worker and metadata client initialized")
    # Update API blueprint references
    if hasattr(app, 'soulsync'):
        app.soulsync['hydrabase_client'] = hydrabase_client
        app.soulsync['hydrabase_worker'] = hydrabase_worker
except Exception as e:
    print(f"⚠️ Hydrabase initialization failed: {e}")
    hydrabase_worker = None
    hydrabase_client = None

# --- Hydrabase Auto-Reconnect ---
try:
    _hydra_cfg = config_manager.get_hydrabase_config()
    if _hydra_cfg.get('auto_connect') and _hydra_cfg.get('url') and _hydra_cfg.get('api_key'):
        import websocket as _ws_mod
        _auto_ws = _ws_mod.create_connection(
            _hydra_cfg['url'],
            header={"x-api-key": _hydra_cfg['api_key']},
            timeout=10
        )
        _hydrabase_ws = _auto_ws
        dev_mode_enabled = True
        print(f"✅ Hydrabase auto-connected to {_hydra_cfg['url']}")
except Exception as e:
    print(f"⚠️ Hydrabase auto-reconnect failed: {e}")

# --- Hydrabase Worker API Endpoints ---

@app.route('/api/hydrabase-worker/status', methods=['GET'])
def hydrabase_worker_status():
    """Get Hydrabase P2P mirror worker status for UI polling"""
    try:
        if hydrabase_worker is None:
            return jsonify({
                'enabled': False,
                'running': False,
                'paused': False,
                'queue_size': 0,
                'stats': {'sent': 0, 'dropped': 0, 'errors': 0}
            }), 200

        status = hydrabase_worker.get_stats()
        return jsonify(status), 200
    except Exception as e:
        logger.error(f"Error getting Hydrabase worker status: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/hydrabase-worker/pause', methods=['POST'])
def hydrabase_worker_pause():
    """Pause Hydrabase P2P mirror worker"""
    try:
        if hydrabase_worker is None:
            return jsonify({'error': 'Hydrabase worker not initialized'}), 400

        hydrabase_worker.pause()
        logger.info("Hydrabase worker paused via UI")
        return jsonify({'status': 'paused'}), 200
    except Exception as e:
        logger.error(f"Error pausing Hydrabase worker: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/hydrabase-worker/resume', methods=['POST'])
def hydrabase_worker_resume():
    """Resume Hydrabase P2P mirror worker"""
    try:
        if hydrabase_worker is None:
            return jsonify({'error': 'Hydrabase worker not initialized'}), 400

        hydrabase_worker.resume()
        logger.info("Hydrabase worker resumed via UI")
        return jsonify({'status': 'running'}), 200
    except Exception as e:
        logger.error(f"Error resuming Hydrabase worker: {e}")
        return jsonify({'error': str(e)}), 500

# ================================================================================================
# END HYDRABASE P2P MIRROR WORKER
# ================================================================================================


# ================================================================================================
# LIBRARY REPAIR WORKER
# ================================================================================================

from core.repair_worker import RepairWorker

repair_worker = None
try:
    from database.music_database import MusicDatabase
    repair_db = MusicDatabase()
    transfer_path = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer'))
    repair_worker = RepairWorker(database=repair_db, transfer_folder=transfer_path)
    repair_worker.start()
    print("✅ Repair worker initialized and started")
except Exception as e:
    print(f"⚠️ Repair worker initialization failed: {e}")
    repair_worker = None

# --- Repair Worker API Endpoints ---

@app.route('/api/repair/status', methods=['GET'])
def repair_status():
    """Get repair worker status for UI polling"""
    try:
        if repair_worker is None:
            return jsonify({
                'enabled': False,
                'running': False,
                'paused': False,
                'current_item': None,
                'stats': {'scanned': 0, 'repaired': 0, 'skipped': 0, 'errors': 0, 'pending': 0},
                'progress': {}
            }), 200

        status = repair_worker.get_stats()
        return jsonify(status), 200
    except Exception as e:
        logger.error(f"Error getting repair status: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/repair/pause', methods=['POST'])
def repair_pause():
    """Pause repair worker"""
    try:
        if repair_worker is None:
            return jsonify({'error': 'Repair worker not initialized'}), 400

        repair_worker.pause()
        logger.info("Repair worker paused via UI")
        return jsonify({'status': 'paused'}), 200
    except Exception as e:
        logger.error(f"Error pausing repair worker: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/repair/resume', methods=['POST'])
def repair_resume():
    """Resume repair worker"""
    try:
        if repair_worker is None:
            return jsonify({'error': 'Repair worker not initialized'}), 400

        repair_worker.resume()
        logger.info("Repair worker resumed via UI")
        return jsonify({'status': 'running'}), 200
    except Exception as e:
        logger.error(f"Error resuming repair worker: {e}")
        return jsonify({'error': str(e)}), 500

# ================================================================================================
# END LIBRARY REPAIR WORKER
# ================================================================================================


# ================================================================================================
# IMPORT / STAGING SYSTEM
# ================================================================================================

AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif', '.ape'}

def _get_staging_path():
    """Get the resolved staging folder path."""
    raw = config_manager.get('import.staging_path', './Staging')
    return docker_resolve_path(raw)


@app.route('/api/import/staging/files', methods=['GET'])
def import_staging_files():
    """Scan the staging folder and return audio files with tag metadata."""
    try:
        staging_path = _get_staging_path()
        os.makedirs(staging_path, exist_ok=True)

        files = []
        for root, _dirs, filenames in os.walk(staging_path):
            for fname in filenames:
                ext = os.path.splitext(fname)[1].lower()
                if ext not in AUDIO_EXTENSIONS:
                    continue
                full_path = os.path.join(root, fname)
                rel_path = os.path.relpath(full_path, staging_path)

                # Try reading tags
                title, artist, album, track_number = None, None, None, None
                try:
                    from mutagen import File as MutagenFile
                    tags = MutagenFile(full_path, easy=True)
                    if tags:
                        title = (tags.get('title') or [None])[0]
                        artist = (tags.get('artist') or [None])[0]
                        album = (tags.get('album') or [None])[0]
                        tn = (tags.get('tracknumber') or [None])[0]
                        if tn:
                            try:
                                track_number = int(str(tn).split('/')[0])
                            except ValueError:
                                pass
                except Exception:
                    pass

                # Fallback to filename parsing
                if not title:
                    parsed = _parse_filename_metadata(fname)
                    title = parsed.get('title') or os.path.splitext(fname)[0]
                    if not artist:
                        artist = parsed.get('artist')
                    if not track_number:
                        track_number = parsed.get('track_number')

                files.append({
                    'filename': fname,
                    'rel_path': rel_path,
                    'full_path': full_path,
                    'title': title,
                    'artist': artist or 'Unknown Artist',
                    'album': album,
                    'track_number': track_number,
                    'extension': ext
                })

        # Sort by filename
        files.sort(key=lambda f: f['filename'].lower())
        return jsonify({'success': True, 'files': files, 'staging_path': staging_path})
    except Exception as e:
        logger.error(f"Error scanning staging files: {e}")
        return jsonify({'success': False, 'error': str(e)}), 500


@app.route('/api/import/staging/hints', methods=['GET'])
def import_staging_hints():
    """Extract album search hints from staging folder (tags + folder names). Fast — no Spotify calls."""
    try:
        staging_path = _get_staging_path()
        if not os.path.isdir(staging_path):
            return jsonify({'success': True, 'hints': []})

        # Collect hints from tags and folder structure
        tag_albums = {}   # (album, artist) -> file count
        folder_hints = {} # subfolder name -> file count

        for root, _dirs, filenames in os.walk(staging_path):
            audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS]
            if not audio_files:
                continue

            # Folder-based hint: use immediate subfolder name relative to staging
            rel_dir = os.path.relpath(root, staging_path)
            if rel_dir != '.':
                # Use the top-level subfolder as the hint
                top_folder = rel_dir.split(os.sep)[0]
                folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files)

            # Tag-based hints
            for fname in audio_files:
                full_path = os.path.join(root, fname)
                try:
                    from mutagen import File as MutagenFile
                    tags = MutagenFile(full_path, easy=True)
                    if tags:
                        album = (tags.get('album') or [None])[0]
                        artist = (tags.get('artist') or (tags.get('albumartist') or [None]))[0]
                        if album:
                            key = (album.strip(), (artist or '').strip())
                            tag_albums[key] = tag_albums.get(key, 0) + 1
                except Exception:
                    pass

        # Build search queries, prioritizing tag-based hints (more specific)
        queries = []
        seen_queries_lower = set()

        # Tag-based: sort by file count descending
        for (album, artist), count in sorted(tag_albums.items(), key=lambda x: -x[1]):
            q = f"{album} {artist}".strip() if artist else album
            if q.lower() not in seen_queries_lower:
                seen_queries_lower.add(q.lower())
                queries.append(q)

        # Folder-based: parse "Artist - Album" pattern or use as-is
        for folder, count in sorted(folder_hints.items(), key=lambda x: -x[1]):
            q = folder.replace('_', ' ')
            if q.lower() not in seen_queries_lower:
                seen_queries_lower.add(q.lower())
                queries.append(q)

        # Cap at 5 queries to keep it fast
        queries = queries[:5]

        return jsonify({'success': True, 'hints': queries})
    except Exception as e:
        logger.error(f"Error getting staging hints: {e}")
        return jsonify({'success': False, 'error': str(e)}), 500


@app.route('/api/import/search/albums', methods=['GET'])
def import_search_albums():
    """Search for albums via Spotify for import matching."""
    try:
        query = request.args.get('q', '').strip()
        if not query:
            return jsonify({'success': False, 'error': 'Missing query parameter'}), 400

        limit = min(int(request.args.get('limit', 12)), 50)

        if _is_hydrabase_active():
            albums = hydrabase_client.search_albums(query, limit=limit)
        else:
            if hydrabase_worker and dev_mode_enabled:
                hydrabase_worker.enqueue(query, 'album')
            albums = spotify_client.search_albums(query, limit=limit)

        results = []
        for a in albums:
            results.append({
                'id': a.id,
                'name': a.name,
                'artist': ', '.join(a.artists) if a.artists else 'Unknown Artist',
                'release_date': a.release_date or '',
                'total_tracks': a.total_tracks,
                'image_url': a.image_url,
                'album_type': a.album_type or 'album'
            })

        return jsonify({'success': True, 'albums': results})
    except Exception as e:
        logger.error(f"Error searching albums for import: {e}")
        return jsonify({'success': False, 'error': str(e)}), 500


@app.route('/api/import/album/match', methods=['POST'])
def import_album_match():
    """Match staging files to an album's tracklist."""
    try:
        data = request.get_json()
        album_id = data.get('album_id')
        album_name = data.get('album_name', '')
        album_artist = data.get('album_artist', '')
        if not album_id:
            return jsonify({'success': False, 'error': 'Missing album_id'}), 400

        spotify_tracks = None
        album_info = None

        # Try Hydrabase first when active — look up by album soul_id
        if _is_hydrabase_active():
            try:
                hydra_tracks = hydrabase_client.get_album_tracks(album_id, limit=50)
                if hydra_tracks:
                    spotify_tracks = []
                    for t in hydra_tracks:
                        artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else []
                        spotify_tracks.append({
                            'name': t.name,
                            'track_number': t.track_number or 0,
                            'disc_number': t.disc_number or 1,
                            'duration_ms': t.duration_ms,
                            'id': t.id,
                            'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list],
                            'uri': ''
                        })
                    album_info = {
                        'id': album_id,
                        'name': album_name or hydra_tracks[0].album or '',
                        'artist': album_artist,
                        'artists': [album_artist] if album_artist else [],
                        'release_date': '',
                        'total_tracks': len(spotify_tracks),
                        'image_url': None,
                        'genres': []
                    }
                    logger.info(f"Hydrabase album_tracks returned {len(spotify_tracks)} tracks for album_id '{album_id}'")
            except Exception as e:
                logger.warning(f"Hydrabase album_tracks failed, falling back to Spotify: {e}")
                spotify_tracks = None

        # Fall back to Spotify
        if spotify_tracks is None:
            album_data = spotify_client.get_album(album_id)
            if not album_data:
                return jsonify({'success': False, 'error': 'Album not found'}), 404

            tracks_data = spotify_client.get_album_tracks(album_id)
            if not tracks_data or 'items' not in tracks_data:
                return jsonify({'success': False, 'error': 'Could not get album tracks'}), 500

            spotify_tracks = tracks_data['items']

            # Build album summary
            album_artists = [a['name'] for a in album_data.get('artists', [])]
            album_info = {
                'id': album_id,
                'name': album_data.get('name', 'Unknown Album'),
                'artist': ', '.join(album_artists),
                'artists': album_artists,
                'release_date': album_data.get('release_date', ''),
                'total_tracks': album_data.get('total_tracks', len(spotify_tracks)),
                'image_url': (album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None),
                'genres': album_data.get('genres', [])
            }

            # Get artist info for context building later
            if album_data.get('artists'):
                primary_artist = album_data['artists'][0]
                album_info['artist_id'] = primary_artist.get('id', '')

        # Scan staging files
        staging_path = _get_staging_path()
        staging_files = []
        for root, _dirs, filenames in os.walk(staging_path):
            for fname in filenames:
                ext = os.path.splitext(fname)[1].lower()
                if ext not in AUDIO_EXTENSIONS:
                    continue
                full_path = os.path.join(root, fname)

                title, artist, album_tag, track_number = None, None, None, None
                try:
                    from mutagen import File as MutagenFile
                    tags = MutagenFile(full_path, easy=True)
                    if tags:
                        title = (tags.get('title') or [None])[0]
                        artist = (tags.get('artist') or [None])[0]
                        album_tag = (tags.get('album') or [None])[0]
                        tn = (tags.get('tracknumber') or [None])[0]
                        if tn:
                            try:
                                track_number = int(str(tn).split('/')[0])
                            except ValueError:
                                pass
                except Exception:
                    pass

                if not title:
                    parsed = _parse_filename_metadata(fname)
                    title = parsed.get('title') or os.path.splitext(fname)[0]
                    if not artist:
                        artist = parsed.get('artist')
                    if not track_number:
                        track_number = parsed.get('track_number')

                staging_files.append({
                    'filename': fname,
                    'full_path': full_path,
                    'title': title,
                    'artist': artist,
                    'album': album_tag,
                    'track_number': track_number
                })

        # Match each Spotify track to the best staging file
        matches = []
        used_files = set()

        for sp_track in spotify_tracks:
            sp_name = sp_track.get('name', '')
            sp_number = sp_track.get('track_number', 0)
            sp_disc = sp_track.get('disc_number', 1)

            best_match = None
            best_score = 0.0

            for i, sf in enumerate(staging_files):
                if i in used_files:
                    continue

                score = 0.0
                # Title similarity (weight 0.5)
                title_sim = matching_engine.similarity_score(
                    matching_engine.normalize_string(sp_name),
                    matching_engine.normalize_string(sf['title'] or '')
                )
                score += title_sim * 0.5

                # Track number match (weight 0.5)
                if sf['track_number'] and sp_number:
                    if sf['track_number'] == sp_number:
                        score += 0.5
                    elif abs(sf['track_number'] - sp_number) <= 1:
                        score += 0.2

                if score > best_score and score >= 0.4:
                    best_score = score
                    best_match = i

            if best_match is not None:
                used_files.add(best_match)
                matches.append({
                    'spotify_track': {
                        'name': sp_name,
                        'track_number': sp_number,
                        'disc_number': sp_disc,
                        'duration_ms': sp_track.get('duration_ms', 0),
                        'id': sp_track.get('id', ''),
                        'artists': [a['name'] for a in sp_track.get('artists', [])],
                        'uri': sp_track.get('uri', '')
                    },
                    'staging_file': staging_files[best_match],
                    'confidence': round(best_score, 2)
                })
            else:
                matches.append({
                    'spotify_track': {
                        'name': sp_name,
                        'track_number': sp_number,
                        'disc_number': sp_disc,
                        'duration_ms': sp_track.get('duration_ms', 0),
                        'id': sp_track.get('id', ''),
                        'artists': [a['name'] for a in sp_track.get('artists', [])],
                        'uri': sp_track.get('uri', '')
                    },
                    'staging_file': None,
                    'confidence': 0
                })

        # Unmatched staging files
        unmatched_files = [sf for i, sf in enumerate(staging_files) if i not in used_files]

        return jsonify({
            'success': True,
            'album': album_info,
            'matches': matches,
            'unmatched_files': unmatched_files
        })
    except Exception as e:
        logger.error(f"Error matching album for import: {e}")
        return jsonify({'success': False, 'error': str(e)}), 500


@app.route('/api/import/album/process', methods=['POST'])
def import_album_process():
    """Process matched album files through the post-processing pipeline."""
    try:
        data = request.get_json()
        album = data.get('album', {})
        matches = data.get('matches', [])

        if not album or not matches:
            return jsonify({'success': False, 'error': 'Missing album or matches data'}), 400

        processed = 0
        errors = []
        album_name = album.get('name', 'Unknown Album')
        artist_name = album.get('artist', 'Unknown Artist')
        artist_id = album.get('artist_id', '')
        album_id = album.get('id', '')

        # Get artist genres from Spotify if possible
        artist_genres = album.get('genres', [])
        if not artist_genres and artist_id:
            try:
                sp_artist = spotify_client.sp.artist(artist_id) if hasattr(spotify_client, 'sp') and spotify_client.sp else None
                if sp_artist:
                    artist_genres = sp_artist.get('genres', [])
            except Exception:
                pass

        for match in matches:
            staging_file = match.get('staging_file')
            spotify_track = match.get('spotify_track')
            if not staging_file or not spotify_track:
                continue

            file_path = staging_file.get('full_path', '')
            if not os.path.isfile(file_path):
                errors.append(f"File not found: {staging_file.get('filename', '?')}")
                continue

            track_name = spotify_track.get('name', 'Unknown Track')
            track_number = spotify_track.get('track_number', 1)
            disc_number = spotify_track.get('disc_number', 1)
            track_artists = spotify_track.get('artists', [artist_name])

            context_key = f"import_album_{album_id}_{track_number}_{uuid.uuid4().hex[:8]}"
            context = {
                'spotify_artist': {
                    'name': artist_name,
                    'id': artist_id,
                    'genres': artist_genres
                },
                'spotify_album': {
                    'id': album_id,
                    'name': album_name,
                    'release_date': album.get('release_date', ''),
                    'total_tracks': album.get('total_tracks', len(matches)),
                    'image_url': album.get('image_url', '')
                },
                'track_info': {
                    'name': track_name,
                    'id': spotify_track.get('id', ''),
                    'track_number': track_number,
                    'disc_number': disc_number,
                    'duration_ms': spotify_track.get('duration_ms', 0),
                    'artists': [{'name': a} if isinstance(a, str) else a for a in track_artists],
                    'uri': spotify_track.get('uri', '')
                },
                'original_search_result': {
                    'title': track_name,
                    'artist': artist_name,
                    'album': album_name,
                    'track_number': track_number,
                    'disc_number': disc_number,
                    'spotify_clean_title': track_name,
                    'spotify_clean_album': album_name,
                    'artists': [{'name': a} if isinstance(a, str) else a for a in track_artists]
                },
                'is_album_download': True,
                'has_clean_spotify_data': True,
                'has_full_spotify_metadata': True
            }

            try:
                _post_process_matched_download(context_key, context, file_path)
                processed += 1
                logger.info(f"Import processed: {track_number}. {track_name} from {album_name}")
            except Exception as proc_err:
                err_msg = f"{track_name}: {str(proc_err)}"
                errors.append(err_msg)
                logger.error(f"Import processing error: {err_msg}")

        # Trigger library scan
        if web_scan_manager and processed > 0:
            threading.Thread(
                target=lambda: web_scan_manager.request_scan("Import album processed"),
                daemon=True
            ).start()

        add_activity_item("📥", "Album Imported", f"{album_name} by {artist_name} ({processed}/{len(matches)} tracks)", "Now")

        try:
            if automation_engine:
                automation_engine.emit('import_completed', {
                    'track_count': str(processed),
                    'album_name': album_name or '',
                    'artist': artist_name or '',
                })
        except Exception:
            pass

        # Rebuild suggestions cache since staging contents changed
        if processed > 0:
            refresh_import_suggestions_cache()

        return jsonify({
            'success': True,
            'processed': processed,
            'total': len(matches),
            'errors': errors
        })
    except Exception as e:
        logger.error(f"Error processing album import: {e}")
        return jsonify({'success': False, 'error': str(e)}), 500


@app.route('/api/import/search/tracks', methods=['GET'])
def import_search_tracks():
    """Search Spotify for individual tracks (used for manual singles identification)."""
    try:
        query = request.args.get('q', '').strip()
        if not query:
            return jsonify({'success': False, 'error': 'Missing query parameter'}), 400

        limit = min(int(request.args.get('limit', 10)), 30)

        if _is_hydrabase_active():
            tracks = hydrabase_client.search_tracks(query, limit=limit)
        else:
            if hydrabase_worker and dev_mode_enabled:
                hydrabase_worker.enqueue(query, 'track')
            tracks = spotify_client.search_tracks(query, limit=limit)

        results = []
        for t in tracks:
            results.append({
                'id': t.id,
                'name': t.name,
                'artist': ', '.join(t.artists) if hasattr(t, 'artists') and t.artists else 'Unknown Artist',
                'album': t.album if hasattr(t, 'album') else '',
                'album_id': t.album_id if hasattr(t, 'album_id') else '',
                'duration_ms': t.duration_ms if hasattr(t, 'duration_ms') else 0,
                'image_url': t.image_url if hasattr(t, 'image_url') else '',
                'track_number': t.track_number if hasattr(t, 'track_number') else 1,
            })

        return jsonify({'success': True, 'tracks': results})
    except Exception as e:
        logger.error(f"Error searching tracks for import: {e}")
        return jsonify({'success': False, 'error': str(e)}), 500


@app.route('/api/import/singles/process', methods=['POST'])
def import_singles_process():
    """Process individual staging files as singles through the post-processing pipeline."""
    try:
        data = request.get_json()
        files = data.get('files', [])

        if not files:
            return jsonify({'success': False, 'error': 'No files provided'}), 400

        processed = 0
        errors = []

        for file_info in files:
            file_path = file_info.get('full_path', '')
            if not os.path.isfile(file_path):
                errors.append(f"File not found: {file_info.get('filename', '?')}")
                continue

            title = file_info.get('title', '')
            artist = file_info.get('artist', '')
            spotify_override = file_info.get('spotify_override', None)

            # Fallback to filename parsing if no metadata
            if not title and not spotify_override:
                parsed = _parse_filename_metadata(file_info.get('filename', ''))
                title = parsed.get('title', os.path.splitext(file_info.get('filename', 'Unknown'))[0])
                if not artist:
                    artist = parsed.get('artist', '')

            # Search Spotify for rich metadata
            spotify_track_data = None
            spotify_artist_data = None
            spotify_album_data = None

            # If a manual spotify_override is provided, look up that specific track
            if spotify_override and spotify_override.get('id'):
                try:
                    override_id = spotify_override['id']
                    sp_track = spotify_client.sp.track(override_id) if hasattr(spotify_client, 'sp') and spotify_client.sp else None
                    if sp_track:
                        sp_track_artists = sp_track.get('artists', [])
                        spotify_track_data = {
                            'name': sp_track.get('name', ''),
                            'id': override_id,
                            'track_number': sp_track.get('track_number', 1),
                            'disc_number': sp_track.get('disc_number', 1),
                            'duration_ms': sp_track.get('duration_ms', 0),
                            'artists': [{'name': a.get('name', '')} for a in sp_track_artists],
                            'uri': sp_track.get('uri', f"spotify:track:{override_id}")
                        }
                        title = sp_track.get('name', title)
                        artist = sp_track_artists[0].get('name', artist) if sp_track_artists else artist
                        # Get album info
                        sp_album_info = sp_track.get('album', {})
                        if sp_album_info:
                            spotify_album_data = {
                                'id': sp_album_info.get('id', ''),
                                'name': sp_album_info.get('name', ''),
                                'release_date': sp_album_info.get('release_date', ''),
                                'total_tracks': sp_album_info.get('total_tracks', 1),
                                'image_url': (sp_album_info.get('images', [{}])[0].get('url') if sp_album_info.get('images') else '')
                            }
                            album_artists = sp_album_info.get('artists', [])
                            if album_artists:
                                spotify_artist_data = {
                                    'name': album_artists[0].get('name', artist),
                                    'id': album_artists[0].get('id', ''),
                                    'genres': []
                                }
                                try:
                                    sp_a = spotify_client.sp.artist(album_artists[0]['id']) if hasattr(spotify_client, 'sp') and spotify_client.sp else None
                                    if sp_a:
                                        spotify_artist_data['genres'] = sp_a.get('genres', [])
                                except Exception:
                                    pass
                except Exception as override_err:
                    logger.warning(f"Spotify override lookup failed for track {spotify_override.get('id')}: {override_err}")

            if not spotify_track_data and title:
                try:
                    search_q = f"{title} {artist}" if artist else title
                    tracks = spotify_client.search_tracks(search_q, limit=1)
                    if tracks:
                        t = tracks[0]
                        spotify_track_data = {
                            'name': t.name,
                            'id': t.id,
                            'track_number': t.track_number if hasattr(t, 'track_number') else 1,
                            'disc_number': 1,
                            'duration_ms': t.duration_ms if hasattr(t, 'duration_ms') else 0,
                            'artists': [{'name': a} for a in (t.artists if hasattr(t, 'artists') else [artist])],
                            'uri': f"spotify:track:{t.id}"
                        }
                        # Get album info from the track's album
                        if hasattr(t, 'album_id') and t.album_id:
                            sp_album = spotify_client.get_album(t.album_id)
                            if sp_album:
                                spotify_album_data = {
                                    'id': t.album_id,
                                    'name': sp_album.get('name', ''),
                                    'release_date': sp_album.get('release_date', ''),
                                    'total_tracks': sp_album.get('total_tracks', 1),
                                    'image_url': (sp_album.get('images', [{}])[0].get('url') if sp_album.get('images') else '')
                                }
                                # Get artist genres
                                sp_artists = sp_album.get('artists', [])
                                if sp_artists:
                                    spotify_artist_data = {
                                        'name': sp_artists[0].get('name', artist),
                                        'id': sp_artists[0].get('id', ''),
                                        'genres': []
                                    }
                                    try:
                                        sp_a = spotify_client.sp.artist(sp_artists[0]['id']) if hasattr(spotify_client, 'sp') and spotify_client.sp else None
                                        if sp_a:
                                            spotify_artist_data['genres'] = sp_a.get('genres', [])
                                    except Exception:
                                        pass

                        # Fallback artist data from track
                        if not spotify_artist_data:
                            track_artists = t.artists if hasattr(t, 'artists') else [artist]
                            spotify_artist_data = {
                                'name': track_artists[0] if track_artists else artist,
                                'id': '',
                                'genres': []
                            }

                        # Fallback album data
                        if not spotify_album_data:
                            spotify_album_data = {
                                'id': '',
                                'name': t.album if hasattr(t, 'album') else '',
                                'release_date': '',
                                'total_tracks': 1,
                                'image_url': t.image_url if hasattr(t, 'image_url') else ''
                            }
                except Exception as sp_err:
                    logger.warning(f"Spotify lookup failed for '{title}': {sp_err}")

            # Build context — use Spotify data if found, else use file metadata
            if not spotify_artist_data:
                spotify_artist_data = {'name': artist or 'Unknown Artist', 'id': '', 'genres': []}
            if not spotify_album_data:
                spotify_album_data = {'id': '', 'name': '', 'release_date': '', 'total_tracks': 1, 'image_url': ''}
            if not spotify_track_data:
                spotify_track_data = {
                    'name': title, 'id': '', 'track_number': 1, 'disc_number': 1,
                    'duration_ms': 0, 'artists': [{'name': artist or 'Unknown Artist'}], 'uri': ''
                }

            final_title = spotify_track_data.get('name', title)
            final_artist = spotify_artist_data.get('name', artist)
            final_album = spotify_album_data.get('name', '')

            context_key = f"import_single_{uuid.uuid4().hex[:8]}"
            context = {
                'spotify_artist': spotify_artist_data,
                'spotify_album': spotify_album_data,
                'track_info': spotify_track_data,
                'original_search_result': {
                    'title': final_title,
                    'artist': final_artist,
                    'album': final_album,
                    'track_number': spotify_track_data.get('track_number', 1),
                    'disc_number': 1,
                    'spotify_clean_title': final_title,
                    'spotify_clean_album': final_album,
                    'artists': spotify_track_data.get('artists', [{'name': final_artist}])
                },
                'is_album_download': False,
                'has_clean_spotify_data': bool(spotify_track_data.get('id')),
                'has_full_spotify_metadata': bool(spotify_track_data.get('id'))
            }

            try:
                _post_process_matched_download(context_key, context, file_path)
                processed += 1
                logger.info(f"Import single processed: {final_title} by {final_artist}")
            except Exception as proc_err:
                err_msg = f"{title}: {str(proc_err)}"
                errors.append(err_msg)
                logger.error(f"Import single processing error: {err_msg}")

        # Trigger library scan
        if web_scan_manager and processed > 0:
            threading.Thread(
                target=lambda: web_scan_manager.request_scan("Import singles processed"),
                daemon=True
            ).start()

        add_activity_item("📥", "Singles Imported", f"{processed}/{len(files)} tracks processed", "Now")

        try:
            if automation_engine:
                automation_engine.emit('import_completed', {
                    'track_count': str(processed),
                    'album_name': '',
                    'artist': 'Various',
                })
        except Exception:
            pass

        # Rebuild suggestions cache since staging contents changed
        if processed > 0:
            refresh_import_suggestions_cache()

        return jsonify({
            'success': True,
            'processed': processed,
            'total': len(files),
            'errors': errors
        })
    except Exception as e:
        logger.error(f"Error processing singles import: {e}")
        return jsonify({'success': False, 'error': str(e)}), 500

# --- Import Suggestion Cache (server-side background builder) ---

_import_suggestions_cache = {
    'suggestions': [],
    'building': False,
    'built': False,
}

def _build_import_suggestions_background():
    """Background thread: extract hints from staging folder, search Spotify, cache results."""
    cache = _import_suggestions_cache
    if cache['building']:
        return
    cache['building'] = True

    try:
        staging_path = _get_staging_path()
        if not os.path.isdir(staging_path):
            cache['suggestions'] = []
            cache['built'] = True
            cache['building'] = False
            return

        # Reuse the hint extraction logic
        tag_albums = {}
        folder_hints = {}

        for root, _dirs, filenames in os.walk(staging_path):
            audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS]
            if not audio_files:
                continue
            rel_dir = os.path.relpath(root, staging_path)
            if rel_dir != '.':
                top_folder = rel_dir.split(os.sep)[0]
                folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files)
            for fname in audio_files:
                full_path = os.path.join(root, fname)
                try:
                    from mutagen import File as MutagenFile
                    tags = MutagenFile(full_path, easy=True)
                    if tags:
                        album = (tags.get('album') or [None])[0]
                        artist = (tags.get('artist') or (tags.get('albumartist') or [None]))[0]
                        if album:
                            key = (album.strip(), (artist or '').strip())
                            tag_albums[key] = tag_albums.get(key, 0) + 1
                except Exception:
                    pass

        queries = []
        seen_lower = set()
        for (album, artist), _ in sorted(tag_albums.items(), key=lambda x: -x[1]):
            q = f"{album} {artist}".strip() if artist else album
            if q.lower() not in seen_lower:
                seen_lower.add(q.lower())
                queries.append(q)
        for folder, _ in sorted(folder_hints.items(), key=lambda x: -x[1]):
            q = folder.replace('_', ' ')
            if q.lower() not in seen_lower:
                seen_lower.add(q.lower())
                queries.append(q)
        queries = queries[:5]

        if not queries:
            cache['suggestions'] = []
            cache['built'] = True
            cache['building'] = False
            return

        suggestions = []
        seen_ids = set()
        for q in queries:
            try:
                if not spotify_client:
                    break
                albums = spotify_client.search_albums(q, limit=2)
                for a in albums:
                    if a.id not in seen_ids:
                        seen_ids.add(a.id)
                        suggestions.append({
                            'id': a.id,
                            'name': a.name,
                            'artist': ', '.join(a.artists) if a.artists else 'Unknown Artist',
                            'release_date': a.release_date or '',
                            'total_tracks': a.total_tracks,
                            'image_url': a.image_url,
                            'album_type': a.album_type or 'album',
                        })
            except Exception as e:
                logger.warning(f"Import suggestion search failed for '{q}': {e}")

        cache['suggestions'] = suggestions[:8]
        cache['built'] = True
        logger.info(f"Import suggestions cache built: {len(cache['suggestions'])} suggestions from {len(queries)} hints")
    except Exception as e:
        logger.error(f"Error building import suggestions cache: {e}")
        cache['suggestions'] = []
        cache['built'] = True
    finally:
        cache['building'] = False


def start_import_suggestions_cache():
    """Start building the import suggestions cache in a background thread (called on server startup)."""
    threading.Thread(
        target=_build_import_suggestions_background,
        daemon=True,
        name='import-suggestions-cache'
    ).start()


def refresh_import_suggestions_cache():
    """Invalidate and rebuild the suggestions cache (called after imports change staging contents)."""
    _import_suggestions_cache['built'] = False
    threading.Thread(
        target=_build_import_suggestions_background,
        daemon=True,
        name='import-suggestions-cache'
    ).start()


@app.route('/api/import/staging/suggestions', methods=['GET'])
def import_staging_suggestions():
    """Return cached import suggestions. If cache isn't built yet, returns partial/empty with a flag."""
    cache = _import_suggestions_cache
    return jsonify({
        'success': True,
        'suggestions': cache['suggestions'],
        'ready': cache['built'],
    })


# ================================================================================================
# END IMPORT / STAGING SYSTEM
# ================================================================================================


# ================================================================================================
# WEBSOCKET (SOCKET.IO) EVENT HANDLERS AND BACKGROUND EMITTERS
# ================================================================================================

def _build_status_payload():
    """Build the same status payload used by GET /status, reading from the cache."""
    download_mode = config_manager.get('download_source.mode', 'soulseek')
    soulseek_data = dict(_status_cache.get('soulseek', {}))
    soulseek_data['source'] = download_mode
    return {
        'spotify': _status_cache.get('spotify', {}),
        'media_server': _status_cache.get('media_server', {}),
        'soulseek': soulseek_data,
        'active_media_server': config_manager.get_active_media_server()
    }

def _build_watchlist_count_payload(profile_id=1):
    """Build the same payload used by GET /api/watchlist/count."""
    try:
        database = get_database()
        count = database.get_watchlist_count(profile_id=profile_id)
    except Exception:
        count = 0
    next_run_in_seconds = automation_engine.get_system_automation_next_run_seconds('scan_watchlist') if automation_engine else 0
    return {
        'success': True,
        'count': count,
        'next_run_in_seconds': next_run_in_seconds
    }

def _emit_service_status_loop():
    """Background thread that pushes service status every 10 seconds."""
    while True:
        socketio.sleep(10)
        try:
            socketio.emit('status:update', _build_status_payload())
        except Exception as e:
            logger.debug(f"Error emitting service status: {e}")

def _emit_watchlist_count_loop():
    """Background thread that pushes watchlist count every 30 seconds to each profile room."""
    while True:
        socketio.sleep(30)
        try:
            database = get_database()
            profiles = database.get_all_profiles()
            for profile in profiles:
                pid = profile['id']
                socketio.emit('watchlist:count', _build_watchlist_count_payload(profile_id=pid), room=f'profile:{pid}')
        except Exception as e:
            logger.debug(f"Error emitting watchlist count: {e}")

def _emit_download_status_loop():
    """Background thread that pushes download batch status every 2 seconds to subscribed rooms."""
    while True:
        socketio.sleep(2)
        try:
            live_transfers_lookup = get_cached_transfer_data()
            with tasks_lock:
                for batch_id, batch in download_batches.items():
                    try:
                        status_data = _build_batch_status_data(
                            batch_id, batch, live_transfers_lookup
                        )
                        socketio.emit('downloads:batch_update', {
                            'batch_id': batch_id,
                            'data': status_data
                        }, room=f'batch:{batch_id}')
                    except Exception as e:
                        logger.debug(f"Error building batch status for {batch_id}: {e}")
        except Exception as e:
            logger.debug(f"Error in download status emit loop: {e}")

# --- Socket.IO event handlers ---

@socketio.on('connect')
def handle_connect():
    logger.info("WebSocket client connected")

@socketio.on('disconnect')
def handle_disconnect():
    logger.info("WebSocket client disconnected")

@socketio.on('downloads:subscribe')
def handle_download_subscribe(data):
    """Client subscribes to download batch updates by joining rooms."""
    batch_ids = data.get('batch_ids', [])
    for bid in batch_ids:
        join_room(f'batch:{bid}')
    logger.debug(f"Client subscribed to batches: {batch_ids}")

@socketio.on('downloads:unsubscribe')
def handle_download_unsubscribe(data):
    """Client unsubscribes from download batch updates by leaving rooms."""
    batch_ids = data.get('batch_ids', [])
    for bid in batch_ids:
        leave_room(f'batch:{bid}')
    logger.debug(f"Client unsubscribed from batches: {batch_ids}")

@socketio.on('profile:join')
def handle_profile_join(data):
    """Client joins a profile room for scoped WebSocket emits (watchlist/wishlist counts)."""
    profile_id = data.get('profile_id')
    if profile_id:
        # Leave any previous profile rooms
        old_id = data.get('old_profile_id')
        if old_id:
            leave_room(f'profile:{old_id}')
        join_room(f'profile:{profile_id}')
        logger.debug(f"Client joined profile room: profile:{profile_id}")

# --- Phase 2: Dashboard emitters ---

def _emit_system_stats_loop():
    """Background thread that pushes system stats every 10 seconds."""
    while True:
        socketio.sleep(10)
        try:
            socketio.emit('dashboard:stats', _build_system_stats())
        except Exception as e:
            logger.debug(f"Error emitting system stats: {e}")

def _emit_activity_feed_loop():
    """Background thread that pushes activity feed every 5 seconds."""
    while True:
        socketio.sleep(5)
        try:
            with activity_feed_lock:
                activities = activity_feed[-10:][::-1]
            socketio.emit('dashboard:activity', {'activities': activities})
        except Exception as e:
            logger.debug(f"Error emitting activity feed: {e}")

def _emit_db_stats_loop():
    """Background thread that pushes database stats every 30 seconds."""
    while True:
        socketio.sleep(30)
        try:
            db = get_database()
            stats = db.get_database_info_for_server()
            socketio.emit('dashboard:db_stats', stats)
        except Exception as e:
            logger.debug(f"Error emitting db stats: {e}")

def _emit_wishlist_count_loop():
    """Background thread that pushes wishlist count every 30 seconds to each profile room."""
    while True:
        socketio.sleep(30)
        try:
            from core.wishlist_service import get_wishlist_service
            ws = get_wishlist_service()
            database = get_database()
            profiles = database.get_all_profiles()
            for profile in profiles:
                pid = profile['id']
                count = ws.get_wishlist_count(profile_id=pid)
                socketio.emit('dashboard:wishlist_count', {'count': count}, room=f'profile:{pid}')
        except Exception as e:
            logger.debug(f"Error emitting wishlist count: {e}")

# Note: Toasts are NOT on a timer — they emit instantly from add_activity_item()

# --- Phase 3: Enrichment sidebar worker emitters ---

def _emit_enrichment_status_loop():
    """Background thread that pushes all enrichment worker statuses every 2 seconds."""
    workers = {
        'musicbrainz': lambda: mb_worker,
        'audiodb': lambda: audiodb_worker,
        'deezer': lambda: deezer_worker,
        'spotify-enrichment': lambda: spotify_enrichment_worker,
        'itunes-enrichment': lambda: itunes_enrichment_worker,
        'hydrabase': lambda: hydrabase_worker,
        'repair': lambda: repair_worker,
    }
    while True:
        socketio.sleep(2)
        for name, get_worker in workers.items():
            try:
                worker = get_worker()
                if worker is None:
                    continue
                status = worker.get_stats()
                socketio.emit(f'enrichment:{name}', status)
            except Exception as e:
                logger.debug(f"Error emitting {name} status: {e}")

def _emit_tool_progress_loop():
    """Background thread that pushes all tool progress statuses every 1 second."""
    while True:
        socketio.sleep(1)
        # Stream status
        try:
            with stream_lock:
                socketio.emit('tool:stream', {
                    "status": stream_state["status"],
                    "progress": stream_state["progress"],
                    "track_info": stream_state["track_info"],
                    "error_message": stream_state["error_message"]
                })
        except Exception as e:
            logger.debug(f"Error emitting stream status: {e}")
        # Quality Scanner
        try:
            with quality_scanner_lock:
                socketio.emit('tool:quality-scanner', dict(quality_scanner_state))
        except Exception as e:
            logger.debug(f"Error emitting quality scanner status: {e}")
        # Duplicate Cleaner (add computed space_freed_mb)
        try:
            with duplicate_cleaner_lock:
                state_copy = duplicate_cleaner_state.copy()
                state_copy["space_freed_mb"] = duplicate_cleaner_state["space_freed"] / (1024 * 1024)
                socketio.emit('tool:duplicate-cleaner', state_copy)
        except Exception as e:
            logger.debug(f"Error emitting duplicate cleaner status: {e}")
        # Retag
        try:
            with retag_lock:
                socketio.emit('tool:retag', dict(retag_state))
        except Exception as e:
            logger.debug(f"Error emitting retag status: {e}")
        # DB Update
        try:
            with db_update_lock:
                socketio.emit('tool:db-update', dict(db_update_state))
        except Exception as e:
            logger.debug(f"Error emitting db update status: {e}")
        # Metadata Update (match HTTP wrapper: {success, status})
        try:
            state_copy = metadata_update_state.copy()
            if state_copy.get('started_at'):
                state_copy['started_at'] = state_copy['started_at'].isoformat()
            if state_copy.get('completed_at'):
                state_copy['completed_at'] = state_copy['completed_at'].isoformat()
            socketio.emit('tool:metadata', {"success": True, "status": state_copy})
        except Exception as e:
            logger.debug(f"Error emitting metadata status: {e}")
        # Logs (format activity_feed same as HTTP endpoint)
        try:
            with activity_feed_lock:
                recent = activity_feed[-50:][::-1]
                formatted = []
                for a in recent:
                    ts = a.get('time', 'Unknown')
                    icon = a.get('icon', '•')
                    title = a.get('title', 'Activity')
                    sub = a.get('subtitle', '')
                    formatted.append(f"[{ts}] {icon} {title} - {sub}" if sub else f"[{ts}] {icon} {title}")
                if not formatted:
                    formatted = ["No recent activity.", "Sync and download operations..."]
                socketio.emit('tool:logs', {'logs': formatted})
        except Exception as e:
            logger.debug(f"Error emitting logs: {e}")

@socketio.on('sync:subscribe')
def handle_sync_subscribe(data):
    for pid in data.get('playlist_ids', []):
        join_room(f'sync:{pid}')

@socketio.on('sync:unsubscribe')
def handle_sync_unsubscribe(data):
    for pid in data.get('playlist_ids', []):
        leave_room(f'sync:{pid}')

@socketio.on('discovery:subscribe')
def handle_discovery_subscribe(data):
    for pid in data.get('ids', []):
        join_room(f'discovery:{pid}')

@socketio.on('discovery:unsubscribe')
def handle_discovery_unsubscribe(data):
    for pid in data.get('ids', []):
        leave_room(f'discovery:{pid}')

def _emit_sync_progress_loop():
    """Push sync progress to subscribed rooms every 1 second."""
    while True:
        socketio.sleep(1)
        try:
            with sync_lock:
                for pid, state in list(sync_states.items()):
                    try:
                        socketio.emit('sync:progress', {
                            'playlist_id': pid, **state
                        }, room=f'sync:{pid}')
                    except Exception:
                        pass
        except Exception as e:
            logger.debug(f"Error in sync progress loop: {e}")

def _emit_discovery_progress_loop():
    """Push discovery progress to subscribed rooms every 1 second."""
    platform_states = {
        'tidal': lambda: tidal_discovery_states,
        'youtube': lambda: youtube_playlist_states,
        'beatport': lambda: beatport_chart_states,
        'listenbrainz': lambda: listenbrainz_playlist_states,
    }
    while True:
        socketio.sleep(1)
        for platform, get_states in platform_states.items():
            try:
                states_dict = get_states()
                for pid, state in list(states_dict.items()):
                    try:
                        phase = state.get('phase', '')
                        if phase in ('', 'idle'):
                            continue
                        payload = {
                            'platform': platform,
                            'id': pid,
                            'phase': state.get('phase'),
                            'status': state.get('status', 'unknown'),
                            'progress': state.get('discovery_progress', 0),
                            'discovery_progress': state.get('discovery_progress', {}),
                            'spotify_matches': state.get('spotify_matches', 0),
                            'spotify_total': state.get('spotify_total', 0),
                            'results': state.get('discovery_results', state.get('results', [])),
                            'complete': state.get('phase') == 'discovered',
                        }
                        socketio.emit('discovery:progress', payload, room=f'discovery:{pid}')
                    except Exception:
                        pass
            except Exception as e:
                logger.debug(f"Error in {platform} discovery loop: {e}")

def _emit_scan_status_loop():
    """Push watchlist and media scan status every 2 seconds."""
    while True:
        socketio.sleep(2)
        # Watchlist scan
        try:
            state = watchlist_scan_state.copy()
            if state.get('started_at'):
                state['started_at'] = state['started_at'].isoformat()
            if state.get('completed_at'):
                state['completed_at'] = state['completed_at'].isoformat()
            state.pop('results', None)
            socketio.emit('scan:watchlist', {"success": True, **state})
        except Exception as e:
            logger.debug(f"Error emitting watchlist scan: {e}")
        # Media scan
        try:
            if web_scan_manager:
                scan_status = web_scan_manager.get_scan_status()
                socketio.emit('scan:media', {"success": True, "status": scan_status})
        except Exception as e:
            logger.debug(f"Error emitting media scan: {e}")
        # Wishlist stats (auto-processing detection + countdown refresh)
        try:
            next_run = automation_engine.get_system_automation_next_run_seconds('process_wishlist') if automation_engine else 0
            socketio.emit('wishlist:stats', {
                "is_auto_processing": is_wishlist_actually_processing(),
                "next_run_in_seconds": next_run,
            })
        except Exception as e:
            logger.debug(f"Error emitting wishlist stats: {e}")

def _emit_automation_progress_loop():
    """Push automation:progress events every 1 second for running automations."""
    while True:
        socketio.sleep(1)
        try:
            with automation_progress_lock:
                active = {}
                stale = []
                now = datetime.now()
                for aid, state in automation_progress_states.items():
                    if state['status'] == 'running':
                        # Timeout zombie running states after 2 hours
                        try:
                            started = datetime.fromisoformat(state.get('started_at', ''))
                            if (now - started).total_seconds() > 7200:
                                state['status'] = 'error'
                                state['phase'] = 'Timed out'
                                state['finished_at'] = now.isoformat()
                                state['log'].append({'type': 'error', 'text': 'Timed out after 2 hours'})
                                # Emit error state before cleanup so frontend sees it
                                cp = dict(state)
                                cp['log'] = list(state['log'])
                                active[str(aid)] = cp
                                continue
                        except (ValueError, TypeError):
                            pass
                        cp = dict(state)
                        cp['log'] = list(state['log'])
                        active[str(aid)] = cp
                    elif state['status'] in ('finished', 'error') and state.get('finished_at'):
                        # Clean up finished states after 60 seconds (frontend already got final emit)
                        try:
                            finished_time = datetime.fromisoformat(state['finished_at'])
                            if (now - finished_time).total_seconds() > 60:
                                stale.append(aid)
                        except (ValueError, TypeError):
                            stale.append(aid)
                for aid in stale:
                    del automation_progress_states[aid]
            if active:
                socketio.emit('automation:progress', active)
        except Exception as e:
            logger.debug(f"Error emitting automation progress: {e}")

# ================================================================================================
# END WEBSOCKET HANDLERS
# ================================================================================================


if __name__ == '__main__':
    # Initialize logging for web server
    from utils.logging_config import setup_logging
    log_level = config_manager.get('logging.level', 'INFO')
    log_path = config_manager.get('logging.path', 'logs/app.log')
    logger = setup_logging(log_level, log_path)

    print("🚀 Starting SoulSync Web UI Server...")
    print("Open your browser and navigate to http://127.0.0.1:8008")
    
    # Start OAuth callback servers
    print("🔧 Starting OAuth callback servers...")
    start_oauth_callback_servers()
    
    # Startup diagnostics: Check and recover stuck flags
    print("🔍 Running startup diagnostics...")
    stuck_flags_recovered = check_and_recover_stuck_flags()
    if stuck_flags_recovered:
        print("⚠️ Recovered stuck flags from previous session")
    else:
        print("✅ No stuck flags detected - system healthy")

    # Start simple background monitor when server starts
    print("🔧 Starting simple background monitor...")
    start_simple_background_monitor()
    print("✅ Simple background monitor started (includes automatic search cleanup)")

    # Wishlist/watchlist timers are now managed by AutomationEngine system automations

    # Pre-build import suggestions cache in background
    print("🔧 Pre-building import suggestions cache...")
    start_import_suggestions_cache()

    # Initialize app start time for uptime tracking
    import time
    app.start_time = time.time()

    # Register action handlers and start automation engine
    _register_automation_handlers()
    if automation_engine:
        print("🔧 Starting automation engine...")
        automation_engine.start()
        print("✅ Automation engine started")
        try:
            automation_engine.emit('app_started', {})
        except Exception:
            pass

    # Add startup activity
    add_activity_item("🚀", "System Started", "SoulSync Web UI Server initialized", "Now")

    # Add a test activity to verify the system is working
    add_activity_item("🔧", "Debug Test", "Activity feed system test", "Now")

    # Start WebSocket background emitters
    print("🔧 Starting WebSocket background emitters...")
    # Phase 1: Global pollers
    socketio.start_background_task(_emit_service_status_loop)
    socketio.start_background_task(_emit_watchlist_count_loop)
    socketio.start_background_task(_emit_download_status_loop)
    # Phase 2: Dashboard pollers
    socketio.start_background_task(_emit_system_stats_loop)
    socketio.start_background_task(_emit_activity_feed_loop)
    socketio.start_background_task(_emit_db_stats_loop)
    socketio.start_background_task(_emit_wishlist_count_loop)
    # Phase 3: Enrichment sidebar workers
    socketio.start_background_task(_emit_enrichment_status_loop)
    # Phase 4: Tool progress pollers
    socketio.start_background_task(_emit_tool_progress_loop)
    # Phase 5: Sync/discovery progress + scans
    socketio.start_background_task(_emit_sync_progress_loop)
    socketio.start_background_task(_emit_discovery_progress_loop)
    socketio.start_background_task(_emit_scan_status_loop)
    # Phase 6: Automation progress
    socketio.start_background_task(_emit_automation_progress_loop)
    print("✅ WebSocket emitters started (Phase 1-6: global/dashboard/enrichment/tools/sync/automations)")

    socketio.run(app, host='0.0.0.0', port=8008, debug=False, allow_unsafe_werkzeug=True)