diff --git a/core/spotify_client.py b/core/spotify_client.py index 8fd09f9d..0e2c962e 100644 --- a/core/spotify_client.py +++ b/core/spotify_client.py @@ -1098,82 +1098,85 @@ class SpotifyClient: return [] @rate_limited - def search_tracks(self, query: str, limit: int = 10) -> List[Track]: - """Search for tracks - falls back to configured metadata source if Spotify not authenticated""" + def search_tracks(self, query: str, limit: int = 10, allow_fallback: bool = True) -> List[Track]: + """Search for tracks. + + When allow_fallback is True, falls back to the configured metadata source + if Spotify is unavailable or returns an error. + """ cache = get_metadata_cache() + effective_limit = min(limit, 50) # Spotify API max is 50 + + # Check Spotify cache first so cached data remains usable even when + # Spotify is temporarily unavailable or rate limited. + cached_results = cache.get_search_results('spotify', 'track', query, effective_limit) + if cached_results is not None: + tracks = [] + for raw in cached_results: + try: + tracks.append(Track.from_spotify_track(raw)) + except Exception: + pass + if tracks: + return tracks + use_spotify = self.is_spotify_authenticated() if use_spotify: - # Check Spotify cache - effective_limit = min(limit, 50) # Spotify API max is 50 - cached_results = cache.get_search_results('spotify', 'track', query, effective_limit) - if cached_results is not None: + try: + results = self.sp.search(q=query, type='track', limit=effective_limit) tracks = [] - for raw in cached_results: - try: - tracks.append(Track.from_spotify_track(raw)) - except Exception: - pass - if tracks: - return tracks + raw_items = results['tracks']['items'] - # Skip Spotify if globally rate limited — fall through to fallback - if self.is_rate_limited(): - logger.debug(f"Spotify rate limited, skipping track search for: {query}") - use_spotify = False - else: - try: - results = self.sp.search(q=query, type='track', limit=effective_limit) - tracks = [] - raw_items = results['tracks']['items'] + for track_data in raw_items: + track = Track.from_spotify_track(track_data) + tracks.append(track) - for track_data in raw_items: - track = Track.from_spotify_track(track_data) - tracks.append(track) + # Cache individual tracks + search mapping + entries = [(td.get('id'), td) for td in raw_items if td.get('id')] + if entries: + cache.store_entities_bulk('spotify', 'track', entries) + cache.store_search_results('spotify', 'track', query, effective_limit, + [td.get('id') for td in raw_items if td.get('id')]) - # Cache individual tracks + search mapping - entries = [(td.get('id'), td) for td in raw_items if td.get('id')] - if entries: - cache.store_entities_bulk('spotify', 'track', entries) - cache.store_search_results('spotify', 'track', query, effective_limit, - [td.get('id') for td in raw_items if td.get('id')]) + return tracks - return tracks - - except Exception as e: - _detect_and_set_rate_limit(e, 'search_tracks') - logger.error(f"Error searching tracks via Spotify: {e}") - # Fall through to fallback + except Exception as e: + _detect_and_set_rate_limit(e, 'search_tracks') + logger.error(f"Error searching tracks via Spotify: {e}") + # Fall through to fallback # Fallback (iTunes or Deezer — configured in settings) - logger.debug(f"Using {self._fallback_source} fallback for track search: {query}") - return self._fallback.search_tracks(query, limit) + if allow_fallback: + logger.debug(f"Using {self._fallback_source} fallback for track search: {query}") + return self._fallback.search_tracks(query, limit) + return [] @rate_limited - def search_artists(self, query: str, limit: int = 10) -> List[Artist]: - """Search for artists - falls back to configured metadata source if Spotify not authenticated""" + def search_artists(self, query: str, limit: int = 10, allow_fallback: bool = True) -> List[Artist]: + """Search for artists. + + When allow_fallback is True, falls back to the configured metadata source + if Spotify is unavailable or returns an error. + """ cache = get_metadata_cache() + # Check Spotify cache first so cached data remains usable even when + # Spotify is temporarily unavailable or rate limited. + cached_results = cache.get_search_results('spotify', 'artist', query, min(limit, 10)) + if cached_results is not None: + artists = [] + for raw in cached_results: + try: + artists.append(Artist.from_spotify_artist(raw)) + except Exception: + pass + if artists: + query_lower = query.lower().strip() + artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1)) + return artists + use_spotify = self.is_spotify_authenticated() - if use_spotify: - # Check Spotify cache - cached_results = cache.get_search_results('spotify', 'artist', query, min(limit, 10)) - if cached_results is not None: - artists = [] - for raw in cached_results: - try: - artists.append(Artist.from_spotify_artist(raw)) - except Exception: - pass - if artists: - query_lower = query.lower().strip() - artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1)) - return artists - - if self.is_rate_limited(): - logger.debug(f"Spotify rate limited, skipping artist search for: {query}") - use_spotify = False - if use_spotify: try: search_query = f'artist:{query}' if len(query.strip()) <= 4 else query @@ -1204,67 +1207,74 @@ class SpotifyClient: # Fall through to iTunes fallback # Fallback (iTunes or Deezer) - logger.debug(f"Using {self._fallback_source} fallback for artist search: {query}") - artists = self._fallback.search_artists(query, limit) - query_lower = query.lower().strip() - artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1)) - return artists + if allow_fallback: + logger.debug(f"Using {self._fallback_source} fallback for artist search: {query}") + artists = self._fallback.search_artists(query, limit) + query_lower = query.lower().strip() + artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1)) + return artists + return [] @rate_limited - def search_albums(self, query: str, limit: int = 10) -> List[Album]: - """Search for albums - falls back to configured metadata source if Spotify not authenticated""" + def search_albums(self, query: str, limit: int = 10, allow_fallback: bool = True) -> List[Album]: + """Search for albums. + + When allow_fallback is True, falls back to the configured metadata source + if Spotify is unavailable or returns an error. + """ cache = get_metadata_cache() + # Check Spotify cache first so cached data remains usable even when + # Spotify is temporarily unavailable or rate limited. + cached_results = cache.get_search_results('spotify', 'album', query, min(limit, 10)) + if cached_results is not None: + albums = [] + for raw in cached_results: + try: + albums.append(Album.from_spotify_album(raw)) + except Exception: + pass + if albums: + return albums + use_spotify = self.is_spotify_authenticated() if use_spotify: - # Check Spotify cache - cached_results = cache.get_search_results('spotify', 'album', query, min(limit, 10)) - if cached_results is not None: + try: + results = self.sp.search(q=query, type='album', limit=min(limit, 10)) albums = [] - for raw in cached_results: - try: - albums.append(Album.from_spotify_album(raw)) - except Exception: - pass - if albums: - return albums + raw_items = results['albums']['items'] - if use_spotify: - # Skip Spotify if globally rate limited — fall through to fallback - if self.is_rate_limited(): - logger.debug(f"Spotify rate limited, skipping album search for: {query}") - use_spotify = False - else: - try: - results = self.sp.search(q=query, type='album', limit=min(limit, 10)) - albums = [] - raw_items = results['albums']['items'] + for album_data in raw_items: + album = Album.from_spotify_album(album_data) + albums.append(album) - for album_data in raw_items: - album = Album.from_spotify_album(album_data) - albums.append(album) + # Cache individual albums + search mapping (skip if full data already cached) + entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')] + if entries: + cache.store_entities_bulk('spotify', 'album', entries, skip_if_exists=True) + cache.store_search_results('spotify', 'album', query, min(limit, 10), + [ad.get('id') for ad in raw_items if ad.get('id')]) - # Cache individual albums + search mapping (skip if full data already cached) - entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')] - if entries: - cache.store_entities_bulk('spotify', 'album', entries, skip_if_exists=True) - cache.store_search_results('spotify', 'album', query, min(limit, 10), - [ad.get('id') for ad in raw_items if ad.get('id')]) + return albums - return albums - - except Exception as e: - _detect_and_set_rate_limit(e, 'search_albums') - logger.error(f"Error searching albums via Spotify: {e}") - # Fall through to iTunes fallback + except Exception as e: + _detect_and_set_rate_limit(e, 'search_albums') + logger.error(f"Error searching albums via Spotify: {e}") + # Fall through to iTunes fallback # Fallback (iTunes or Deezer) - logger.debug(f"Using {self._fallback_source} fallback for album search: {query}") - return self._fallback.search_albums(query, limit) + if allow_fallback: + logger.debug(f"Using {self._fallback_source} fallback for album search: {query}") + return self._fallback.search_albums(query, limit) + return [] @rate_limited - def get_track_details(self, track_id: str) -> Optional[Dict[str, Any]]: - """Get detailed track information - falls back to configured metadata source""" + def get_track_details(self, track_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]: + """Get detailed track information. + + When allow_fallback is True, falls back to the configured metadata source + for non-Spotify IDs or Spotify failure. + """ # Check cache — we store raw track_data, reconstruct enhanced on hit cache = get_metadata_cache() fallback_src = self._fallback_source @@ -1277,7 +1287,7 @@ class SpotifyClient: return self._build_enhanced_track(cached) # Simplified track cached by get_album_tracks — treat as cache miss logger.debug(f"Cache hit for track {track_id} lacks album data, fetching full data") - else: + elif allow_fallback: # Fallback cache hit — delegate to fallback client which reconstructs enhanced format return self._fallback.get_track_details(track_id) @@ -1298,7 +1308,7 @@ class SpotifyClient: # Fall through to iTunes fallback # Fallback - only if ID is numeric (non-Spotify format) - if self._is_itunes_id(track_id): + if allow_fallback and self._is_itunes_id(track_id): logger.debug(f"Using {fallback_src} fallback for track details: {track_id}") result = self._fallback.get_track_details(track_id) return result @@ -1354,8 +1364,12 @@ class SpotifyClient: return None @rate_limited - def get_album(self, album_id: str) -> Optional[Dict[str, Any]]: - """Get album information - falls back to configured metadata source""" + def get_album(self, album_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]: + """Get album information. + + When allow_fallback is True, falls back to the configured metadata source + for non-Spotify IDs or Spotify failure. + """ # Check cache first cache = get_metadata_cache() fallback_src = self._fallback_source @@ -1368,7 +1382,7 @@ class SpotifyClient: return cached # Simplified album cached by get_artist_albums — treat as cache miss logger.debug(f"Cache hit for album {album_id} lacks tracks, fetching full data") - else: + elif allow_fallback: # Fallback cache hit — delegate to fallback client return self._fallback.get_album(album_id) @@ -1385,7 +1399,7 @@ class SpotifyClient: # Fall through to fallback # Fallback - only if ID is numeric (non-Spotify format) - if self._is_itunes_id(album_id): + if allow_fallback and self._is_itunes_id(album_id): logger.debug(f"Using {fallback_src} fallback for album: {album_id}") return self._fallback.get_album(album_id) else: @@ -1393,8 +1407,12 @@ class SpotifyClient: return None @rate_limited - def get_album_tracks(self, album_id: str) -> Optional[Dict[str, Any]]: - """Get album tracks - falls back to configured metadata source""" + def get_album_tracks(self, album_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]: + """Get album tracks. + + When allow_fallback is True, falls back to the configured metadata source + for non-Spotify IDs or Spotify failure. + """ # Cache key uses album_id with '_tracks' suffix to differentiate from album metadata cache = get_metadata_cache() fallback_src = self._fallback_source @@ -1458,7 +1476,7 @@ class SpotifyClient: # Fall through to iTunes fallback # Fallback - only if ID is numeric (non-Spotify format) - if self._is_itunes_id(album_id): + if allow_fallback and self._is_itunes_id(album_id): logger.debug(f"Using {fallback_src} fallback for album tracks: {album_id}") result = self._fallback.get_album_tracks(album_id) return result @@ -1467,8 +1485,12 @@ class SpotifyClient: return None @rate_limited - def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 10, skip_cache: bool = False, max_pages: int = 0) -> List[Album]: - """Get albums by artist ID - falls back to iTunes if Spotify not authenticated. + def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 10, + skip_cache: bool = False, max_pages: int = 0, allow_fallback: bool = True) -> List[Album]: + """Get albums by artist ID. + + When allow_fallback is True, falls back to iTunes/Deezer if Spotify + is not authenticated or errors. Set skip_cache=True for watchlist scans that need fresh data to detect new releases. Set max_pages to limit pagination (0 = fetch all). Spotify returns newest first, so max_pages=1 is sufficient for new release detection.""" @@ -1540,7 +1562,7 @@ class SpotifyClient: # Fall through to iTunes fallback # Fallback - only if ID is numeric (non-Spotify format) - if self._is_itunes_id(artist_id): + if allow_fallback and self._is_itunes_id(artist_id): logger.debug(f"Using {fallback_src} fallback for artist albums: {artist_id}") return self._fallback.get_artist_albums(artist_id, album_type, limit) else: @@ -1559,9 +1581,9 @@ class SpotifyClient: return None @rate_limited - def get_artist(self, artist_id: str) -> Optional[Dict[str, Any]]: + def get_artist(self, artist_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]: """ - Get full artist details - falls back to configured metadata source. + Get full artist details. Args: artist_id: Artist ID (Spotify or fallback source depending on authentication) @@ -1577,8 +1599,10 @@ class SpotifyClient: if cached: if source == 'spotify': return cached # Spotify raw format is the expected format - # Fallback cache hit — delegate to fallback client which reconstructs Spotify-compatible format - return self._fallback.get_artist(artist_id) + if allow_fallback: + # Fallback cache hit — delegate to fallback client which reconstructs Spotify-compatible format + return self._fallback.get_artist(artist_id) + return None if self.is_spotify_authenticated(): try: @@ -1592,7 +1616,7 @@ class SpotifyClient: # Fall through to iTunes fallback # Fallback - only if ID is numeric (non-Spotify format) - if self._is_itunes_id(artist_id): + if allow_fallback and self._is_itunes_id(artist_id): logger.debug(f"Using {fallback_src} fallback for artist: {artist_id}") return self._fallback.get_artist(artist_id) else: diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 0815a528..643d9b93 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -4,7 +4,7 @@ Watchlist Scanner Service - Monitors watched artists for new releases """ -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Callable from datetime import datetime, timezone, timedelta from dataclasses import dataclass import re @@ -13,6 +13,12 @@ import requests from bs4 import BeautifulSoup from database.music_database import get_database, WatchlistArtist from core.spotify_client import SpotifyClient +from core.metadata_service import ( + get_album_tracks_for_source, + get_client_for_source, + get_primary_source, + get_source_priority, +) from core.wishlist_service import get_wishlist_service from core.matching_engine import MusicMatchingEngine from utils.logging_config import get_logger @@ -344,6 +350,15 @@ class ScanResult: success: bool error_message: Optional[str] = None + +@dataclass +class WatchlistDiscographyResult: + """Resolved watchlist artist discography for a specific metadata source.""" + source: str + artist_id: str + albums: List[Any] + image_url: Optional[str] = None + class WatchlistScanner: """Service for scanning watched artists for new releases""" @@ -398,11 +413,6 @@ class WatchlistScanner: self._metadata_service = MetadataService() return self._metadata_service - def _reset_spotify_run_state(self): - """Clear per-run Spotify suppression state.""" - self._spotify_disabled_for_run = False - self._spotify_disabled_reason = None - def _disable_spotify_for_run(self, reason: str): """Disable Spotify for rest of current run, once.""" if not self._spotify_disabled_for_run: @@ -433,476 +443,781 @@ class WatchlistScanner: if not self._spotify_available_for_run(): return False try: - from core.metadata_service import get_primary_source return get_primary_source() == 'spotify' except Exception: return False - def _get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist): - """ - Get the appropriate client and artist ID based on active provider. - If iTunes ID is missing, searches by artist name to find and cache it. + def _watchlist_source_priority(self) -> List[str]: + """Return watchlist scan sources in the configured priority order.""" + return list(get_source_priority(get_primary_source())) - Returns: - Tuple of (client, artist_id, provider_name) or (None, None, None) if no valid ID - """ - provider = self.metadata_service.get_active_provider() + @staticmethod + def _artist_id_attribute_for_source(source: str) -> Optional[str]: + """Return the watchlist artist attribute that stores the given source ID.""" + return { + 'spotify': 'spotify_artist_id', + 'itunes': 'itunes_artist_id', + 'deezer': 'deezer_artist_id', + 'discogs': 'discogs_artist_id', + }.get(source) - if provider == 'spotify': - if watchlist_artist.spotify_artist_id: - return (self.metadata_service.spotify, watchlist_artist.spotify_artist_id, 'spotify') - else: - logger.warning(f"No Spotify ID for {watchlist_artist.artist_name}, cannot scan with Spotify") - return (None, None, None) - else: # itunes or deezer fallback - fallback_source = provider # 'itunes' or 'deezer' - fallback_client = self.metadata_service.itunes # May be iTunesClient or DeezerClient - # Pick the right stored ID for the active fallback source - stored_id = watchlist_artist.deezer_artist_id if fallback_source == 'deezer' else watchlist_artist.itunes_artist_id - if stored_id: - return (fallback_client, stored_id, fallback_source) - else: - # No ID stored for this source - search by name and cache it - logger.info(f"No {fallback_source} ID for {watchlist_artist.artist_name}, searching by name...") - try: - search_results = fallback_client.search_artists(watchlist_artist.artist_name, limit=1) - if search_results and len(search_results) > 0: - found_id = search_results[0].id - logger.info(f"Found {fallback_source} ID {found_id} for {watchlist_artist.artist_name}") - # Cache the ID in the database for future use - if fallback_source == 'deezer': - self.database.update_watchlist_artist_deezer_id( - watchlist_artist.spotify_artist_id or str(watchlist_artist.id), - found_id - ) - else: - self.database.update_watchlist_artist_itunes_id( - watchlist_artist.spotify_artist_id or str(watchlist_artist.id), - found_id - ) - return (fallback_client, found_id, fallback_source) - else: - logger.warning(f"Could not find {watchlist_artist.artist_name} on {fallback_source}") - return (None, None, None) - except Exception as e: - logger.error(f"Error searching {fallback_source} for {watchlist_artist.artist_name}: {e}") - return (None, None, None) + @staticmethod + def _extract_entity_id(value: Any) -> Optional[str]: + """Extract an ID from a dataclass, dict, or plain object.""" + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, dict): + return value.get('id') or value.get('artist_id') or value.get('release_id') + return getattr(value, 'id', None) or getattr(value, 'artist_id', None) or getattr(value, 'release_id', None) - def get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist): - """ - Public wrapper for _get_active_client_and_artist_id. - Gets the appropriate client and artist ID based on active provider. + def _cache_watchlist_artist_source_id(self, watchlist_artist: WatchlistArtist, source: str, source_id: str) -> None: + """Cache a resolved artist ID for a watchlist artist when we have a storage column.""" + if not source_id: + return - Returns: - Tuple of (client, artist_id, provider_name) or (None, None, None) if no valid ID - """ - return self._get_active_client_and_artist_id(watchlist_artist) + if source == 'spotify': + self.database.update_watchlist_spotify_id(watchlist_artist.id, source_id) + watchlist_artist.spotify_artist_id = source_id + elif source == 'itunes': + self.database.update_watchlist_itunes_id(watchlist_artist.id, source_id) + watchlist_artist.itunes_artist_id = source_id + elif source == 'deezer': + self.database.update_watchlist_deezer_id(watchlist_artist.id, source_id) + watchlist_artist.deezer_artist_id = source_id + elif source == 'discogs': + self.database.update_watchlist_discogs_id(watchlist_artist.id, source_id) + watchlist_artist.discogs_artist_id = source_id - def get_artist_image_url(self, watchlist_artist: WatchlistArtist) -> Optional[str]: - """ - Get artist image URL using the active provider. + def _resolve_watchlist_artist_source_id(self, watchlist_artist: WatchlistArtist, source: str, client: Any) -> Optional[str]: + """Resolve the artist ID for an exact source, searching by name if needed.""" + attr = self._artist_id_attribute_for_source(source) + stored_id = getattr(watchlist_artist, attr, None) if attr else None + if stored_id: + return stored_id - Returns: - Image URL string or None if not available - """ - client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist) - if not client or not artist_id: + if not client or not hasattr(client, 'search_artists'): + return None + + try: + search_results = client.search_artists(watchlist_artist.artist_name, limit=1) + except Exception as e: + logger.debug("Could not search %s for %s: %s", source, watchlist_artist.artist_name, e) + return None + + if not search_results: + return None + + found_id = self._extract_entity_id(search_results[0]) + if found_id and attr: + self._cache_watchlist_artist_source_id(watchlist_artist, source, found_id) + return found_id + + @staticmethod + def _get_artist_image_from_data(artist_data: Any) -> Optional[str]: + """Extract an image URL from artist payloads across providers.""" + if not artist_data: + return None + + if isinstance(artist_data, dict): + images = artist_data.get('images') or [] + if images: + first_image = images[0] + if isinstance(first_image, dict): + return first_image.get('url') + return ( + artist_data.get('image_url') + or artist_data.get('thumb_url') + or artist_data.get('cover_image') + or artist_data.get('picture_xl') + or artist_data.get('picture_big') + or artist_data.get('picture_medium') + ) + + images = getattr(artist_data, 'images', None) + if images: + first_image = images[0] + if isinstance(first_image, dict): + return first_image.get('url') + return ( + getattr(artist_data, 'image_url', None) + or getattr(artist_data, 'thumb_url', None) + or getattr(artist_data, 'cover_image', None) + ) + + def _get_artist_image_for_source(self, watchlist_artist: WatchlistArtist, source: str, client: Any, artist_id: str) -> Optional[str]: + """Fetch an artist image for a specific source.""" + if not client or not artist_id or not hasattr(client, 'get_artist'): return None try: artist_data = client.get_artist(artist_id) - if artist_data: - # Handle both Spotify and iTunes response formats - if 'images' in artist_data and artist_data['images']: - return artist_data['images'][0].get('url') - elif 'image_url' in artist_data: - return artist_data['image_url'] except Exception as e: - logger.debug(f"Could not fetch artist image for {watchlist_artist.artist_name}: {e}") + logger.debug("Could not fetch artist image for %s on %s: %s", watchlist_artist.artist_name, source, e) + return None + return self._get_artist_image_from_data(artist_data) + + def _get_album_data_for_source(self, source: str, album_id: str, album_name: str = '') -> Optional[Dict[str, Any]]: + """Fetch album data for a specific source and normalize track payloads when needed.""" + client = get_client_for_source(source) + if not client or not album_id or not hasattr(client, 'get_album'): + return None + + try: + album_data = client.get_album(album_id) + except Exception as e: + logger.debug("Could not fetch album %s on %s: %s", album_id, source, e) + album_data = None + + if not album_data: + return None + + # Some providers return album metadata without embedded tracks; normalize that shape. + tracks = album_data.get('tracks') if isinstance(album_data, dict) else None + if not tracks: + track_items = get_album_tracks_for_source(source, album_id) + if track_items: + if not isinstance(album_data, dict): + try: + album_data = dict(album_data) + except Exception: + album_data = {'name': album_name or album_id} + if isinstance(track_items, dict): + album_data['tracks'] = track_items + else: + album_data['tracks'] = {'items': track_items} + + return album_data + + @staticmethod + def _extract_track_items(album_data: Any) -> List[Dict[str, Any]]: + """Normalize track payloads from different album formats to a list of items.""" + if not album_data: + return [] + + tracks = None + if isinstance(album_data, dict): + tracks = album_data.get('tracks') + else: + tracks = getattr(album_data, 'tracks', None) + + if not tracks: + return [] + + if isinstance(tracks, dict): + items = tracks.get('items') or tracks.get('data') or [] + return list(items) if isinstance(items, list) else [] + + if isinstance(tracks, list): + return tracks + + return [] + + def _resolve_watchlist_discography_for_source( + self, + watchlist_artist: WatchlistArtist, + source: str, + last_scan_timestamp: Optional[datetime] = None, + ) -> Optional[WatchlistDiscographyResult]: + """Resolve a watchlist artist to a specific source and fetch its discography.""" + client = get_client_for_source(source) + if not client: + return None + + artist_id = self._resolve_watchlist_artist_source_id(watchlist_artist, source, client) + if not artist_id: + return None + + albums = self._get_artist_discography_with_client( + client, + artist_id, + last_scan_timestamp, + lookback_days=watchlist_artist.lookback_days, + ) + if not albums: + return None + + image_url = self._get_artist_image_for_source(watchlist_artist, source, client, artist_id) + return WatchlistDiscographyResult( + source=source, + artist_id=artist_id, + albums=albums, + image_url=image_url, + ) + + def get_artist_image_url(self, watchlist_artist: WatchlistArtist) -> Optional[str]: + """ + Get artist image URL using the configured source priority. + + Returns: + Image URL string or None if not available + """ + for source in self._watchlist_source_priority(): + client = get_client_for_source(source) + if not client: + continue + artist_id = self._resolve_watchlist_artist_source_id(watchlist_artist, source, client) + if not artist_id: + continue + image_url = self._get_artist_image_for_source(watchlist_artist, source, client, artist_id) + if image_url: + return image_url return None - def get_artist_discography_for_watchlist(self, watchlist_artist: WatchlistArtist, last_scan_timestamp: Optional[datetime] = None) -> Optional[List]: + def backfill_watchlist_artist_images(self, profile_id: int) -> int: + """Backfill missing watchlist artist images using cached metadata and existing album art.""" + try: + conn = self.database._get_connection() + cursor = conn.cursor() + cursor.execute(""" + SELECT id, artist_name, spotify_artist_id, itunes_artist_id, + deezer_artist_id, discogs_artist_id + FROM watchlist_artists + WHERE profile_id = ? AND (image_url IS NULL OR image_url = '' OR image_url = 'None' + OR image_url NOT LIKE 'http%') + """, (profile_id,)) + imageless = cursor.fetchall() + + if not imageless: + return 0 + + logger.info("Backfilling images for %s watchlist artists (profile %s)...", len(imageless), profile_id) + filled = 0 + for row in imageless: + name = row['artist_name'] + img = None + + # 1. Check metadata cache for artist image + cursor.execute(""" + SELECT image_url FROM metadata_cache_entities + WHERE entity_type = 'artist' AND name = ? COLLATE NOCASE + AND image_url IS NOT NULL AND image_url LIKE 'http%' + LIMIT 1 + """, (name,)) + cr = cursor.fetchone() + if cr: + img = cr['image_url'] + + # 2. Deezer direct URL (no API call needed) + if not img and row['deezer_artist_id']: + img = f"https://api.deezer.com/artist/{row['deezer_artist_id']}/image?size=big" + + # 3. Deezer ID from cache (artist may have a Deezer match we haven't stored on watchlist) + if not img: + cursor.execute(""" + SELECT entity_id FROM metadata_cache_entities + WHERE entity_type = 'artist' AND source = 'deezer' + AND name = ? COLLATE NOCASE LIMIT 1 + """, (name,)) + dz = cursor.fetchone() + if dz and dz['entity_id']: + img = f"https://api.deezer.com/artist/{dz['entity_id']}/image?size=big" + + # 4. Album art fallback (iTunes artists have no artist images) + if not img: + cursor.execute(""" + SELECT image_url FROM metadata_cache_entities + WHERE entity_type = 'album' AND image_url LIKE 'http%' + AND artist_name = ? COLLATE NOCASE LIMIT 1 + """, (name,)) + alb = cursor.fetchone() + if alb: + img = alb['image_url'] + + if img: + aid = (row['spotify_artist_id'] or row['itunes_artist_id'] + or row['deezer_artist_id'] or row['discogs_artist_id']) + if aid: + self.database.update_watchlist_artist_image(aid, img) + else: + # No external IDs — update by internal row ID directly + cursor.execute(""" + UPDATE watchlist_artists SET image_url = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, (img, row['id'])) + conn.commit() + filled += 1 + + if filled: + logger.info("Backfilled %s/%s watchlist artist images (profile %s)", filled, len(imageless), profile_id) + return filled + except Exception as e: + logger.debug("Error backfilling watchlist artist images for profile %s: %s", profile_id, e, exc_info=True) + return 0 + + def get_artist_discography_for_watchlist(self, watchlist_artist: WatchlistArtist, last_scan_timestamp: Optional[datetime] = None) -> Optional[WatchlistDiscographyResult]: """ - Get artist's discography using the active provider, with proper ID resolution. - This is the provider-aware version of get_artist_discography. + Get artist's discography using the configured source priority, with proper ID resolution. + Returns the first provider that can actually return albums. Args: watchlist_artist: WatchlistArtist object (has both spotify and itunes IDs) last_scan_timestamp: Only return releases after this date (for incremental scans) Returns: - List of albums or None on error + WatchlistDiscographyResult or None on error """ - client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist) - if not client or not artist_id: - logger.warning(f"No valid client/ID for {watchlist_artist.artist_name}") - return None + for source in self._watchlist_source_priority(): + result = self._resolve_watchlist_discography_for_source(watchlist_artist, source, last_scan_timestamp) + if result: + return result - albums = self._get_artist_discography_with_client(client, artist_id, last_scan_timestamp, lookback_days=watchlist_artist.lookback_days) - - # If primary provider returned nothing, try the other provider as fallback - if not albums: - fallback_id = None - fallback_client = None - - if provider == 'spotify': - fallback_client = self.metadata_service.itunes - fallback_id = watchlist_artist.itunes_artist_id - # If no iTunes ID stored, search by name and cache it - if not fallback_id: - try: - search_results = fallback_client.search_artists(watchlist_artist.artist_name, limit=1) - if search_results: - fallback_id = search_results[0].id - logger.info(f"Resolved iTunes ID {fallback_id} for {watchlist_artist.artist_name}") - self.database.update_watchlist_artist_itunes_id( - watchlist_artist.spotify_artist_id or str(watchlist_artist.id), - fallback_id - ) - watchlist_artist.itunes_artist_id = fallback_id - except Exception as e: - logger.debug(f"Could not resolve iTunes ID for {watchlist_artist.artist_name}: {e}") - - elif provider == 'itunes': - fallback_client = self.metadata_service.spotify - fallback_id = watchlist_artist.spotify_artist_id - - if fallback_client and fallback_id: - logger.info(f"{provider.capitalize()} returned no albums for {watchlist_artist.artist_name}, falling back to {'iTunes' if provider == 'spotify' else 'Spotify'}") - albums = self._get_artist_discography_with_client(fallback_client, fallback_id, last_scan_timestamp, lookback_days=watchlist_artist.lookback_days) - - return albums - - def scan_all_watchlist_artists(self) -> List[ScanResult]: - """ - Scan artists in the watchlist for new releases. - - OPTIMIZED: Scans up to 50 artists per run using smart selection: - - Priority: Artists not scanned in 7+ days (guaranteed) - - Remainder: Random selection from other artists - - This reduces API calls while ensuring all artists scanned at least weekly. - Only checks releases after their last scan timestamp. - """ - logger.info("Starting watchlist scan") + logger.warning(f"No valid client/ID for {watchlist_artist.artist_name}") + return None + def _apply_global_watchlist_overrides(self, watchlist_artists: List[WatchlistArtist]): + """Apply global watchlist release-type overrides to a batch of artists.""" try: - self._reset_spotify_run_state() - from datetime import datetime, timedelta - import random + from config.settings import config_manager + except Exception: + return - # Get all watchlist artists - all_watchlist_artists = self.database.get_watchlist_artists() - if not all_watchlist_artists: - logger.info("No artists in watchlist to scan") - return [] + if not config_manager.get('watchlist.global_override_enabled', False): + return - logger.info(f"Found {len(all_watchlist_artists)} total artists in watchlist") + g_albums = config_manager.get('watchlist.global_include_albums', True) + g_eps = config_manager.get('watchlist.global_include_eps', True) + g_singles = config_manager.get('watchlist.global_include_singles', True) + g_live = config_manager.get('watchlist.global_include_live', False) + g_remixes = config_manager.get('watchlist.global_include_remixes', False) + g_acoustic = config_manager.get('watchlist.global_include_acoustic', False) + g_compilations = config_manager.get('watchlist.global_include_compilations', False) + g_instrumentals = config_manager.get('watchlist.global_include_instrumentals', False) - # OPTIMIZATION: Select up to 50 artists to scan - # 1. Must scan: Artists not scanned in 7+ days (or never scanned) - seven_days_ago = datetime.now() - timedelta(days=7) - must_scan = [] - can_skip = [] + logger.info( + "Applying global watchlist override to %s artists " + "(albums=%s, eps=%s, singles=%s, live=%s, remixes=%s, acoustic=%s, compilations=%s, instrumentals=%s)", + len(watchlist_artists), + g_albums, + g_eps, + g_singles, + g_live, + g_remixes, + g_acoustic, + g_compilations, + g_instrumentals, + ) - for artist in all_watchlist_artists: - if artist.last_scan_timestamp is None: - # Never scanned - must scan - must_scan.append(artist) - elif artist.last_scan_timestamp < seven_days_ago: - # Not scanned in 7+ days - must scan - must_scan.append(artist) - else: - # Scanned recently - can skip (but might randomly select) - can_skip.append(artist) + for artist in watchlist_artists: + artist.include_albums = g_albums + artist.include_eps = g_eps + artist.include_singles = g_singles + artist.include_live = g_live + artist.include_remixes = g_remixes + artist.include_acoustic = g_acoustic + artist.include_compilations = g_compilations + artist.include_instrumentals = g_instrumentals - logger.info(f"Artists requiring scan (not scanned in 7+ days): {len(must_scan)}") - logger.info(f"Artists scanned recently (< 7 days): {len(can_skip)}") + def scan_watchlist_profile( + self, + profile_id: int, + watchlist_artists: Optional[List[WatchlistArtist]] = None, + *, + scan_state: Optional[Dict[str, Any]] = None, + progress_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, + cancel_check: Optional[Callable[[], bool]] = None, + artist_index_offset: int = 0, + total_artists_override: Optional[int] = None, + apply_global_overrides: bool = True, + ) -> List[ScanResult]: + """Scan a single watchlist profile using the shared watchlist scan engine.""" + if watchlist_artists is None: + watchlist_artists = self.database.get_watchlist_artists(profile_id=profile_id) - # 2. Fill remaining slots (up to 50 total) with random selection - max_artists_per_scan = 50 - artists_to_scan = must_scan.copy() + if apply_global_overrides: + self._apply_global_watchlist_overrides(watchlist_artists) - remaining_slots = max_artists_per_scan - len(must_scan) - if remaining_slots > 0 and can_skip: - # Randomly sample from recently-scanned artists - random_sample_size = min(remaining_slots, len(can_skip)) - random_selection = random.sample(can_skip, random_sample_size) - artists_to_scan.extend(random_selection) - logger.info(f"Additionally scanning {len(random_selection)} randomly selected artists") + return self.scan_watchlist_artists( + watchlist_artists, + profile_id=profile_id, + scan_state=scan_state, + progress_callback=progress_callback, + cancel_check=cancel_check, + artist_index_offset=artist_index_offset, + total_artists_override=total_artists_override, + ) - # Shuffle to avoid always scanning same order - random.shuffle(artists_to_scan) + def scan_watchlist_artists( + self, + watchlist_artists: List[WatchlistArtist], + *, + profile_id: int = 1, + scan_state: Optional[Dict[str, Any]] = None, + progress_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, + cancel_check: Optional[Callable[[], bool]] = None, + artist_index_offset: int = 0, + total_artists_override: Optional[int] = None, + ) -> List[ScanResult]: + """Scan a list of watchlist artists using the shared web watchlist scan flow.""" + scan_results: List[ScanResult] = [] + if not watchlist_artists: + if scan_state is not None: + scan_state.update({ + 'status': 'completed', + 'total_artists': 0, + 'current_artist_index': 0, + 'current_artist_name': '', + 'current_artist_image_url': '', + 'current_phase': 'completed', + 'albums_to_check': 0, + 'albums_checked': 0, + 'current_album': '', + 'current_album_image_url': '', + 'current_track_name': '', + 'tracks_found_this_scan': 0, + 'tracks_added_this_scan': 0, + 'recent_wishlist_additions': [], + 'results': [], + 'summary': { + 'total_artists': 0, + 'successful_scans': 0, + 'new_tracks_found': 0, + 'tracks_added_to_wishlist': 0, + }, + 'completed_at': datetime.now(), + 'error': None, + }) + return scan_results - logger.info(f"Total artists to scan this run: {len(artists_to_scan)}") - if len(all_watchlist_artists) > max_artists_per_scan: - logger.info(f"Skipping {len(all_watchlist_artists) - len(artists_to_scan)} artists (will be scanned in future runs)") + if scan_state is not None: + scan_state.update({ + 'status': 'scanning', + 'started_at': scan_state.get('started_at') or datetime.now(), + 'total_artists': total_artists_override if total_artists_override is not None else len(watchlist_artists), + 'current_artist_index': scan_state.get('current_artist_index', artist_index_offset), + 'current_artist_name': scan_state.get('current_artist_name', ''), + 'current_artist_image_url': scan_state.get('current_artist_image_url', ''), + 'current_phase': 'starting', + 'albums_to_check': 0, + 'albums_checked': 0, + 'current_album': '', + 'current_album_image_url': '', + 'current_track_name': '', + 'tracks_found_this_scan': scan_state.get('tracks_found_this_scan', 0), + 'tracks_added_this_scan': scan_state.get('tracks_added_this_scan', 0), + 'recent_wishlist_additions': scan_state.get('recent_wishlist_additions', []), + 'results': scan_state.get('results', []), + 'summary': scan_state.get('summary', {}), + 'error': None, + }) - watchlist_artists = artists_to_scan - - # PROACTIVE ID BACKFILLING (cross-provider support) - # Before scanning, ensure ALL artists have IDs for ALL available sources - # iTunes and Deezer are always available; Spotify requires authentication - if self.spotify_client and self.spotify_client.is_rate_limited(): - self._disable_spotify_for_run("global Spotify rate limit active") - providers_to_backfill = ['itunes', 'deezer'] - if self._spotify_is_primary_source(): - providers_to_backfill.append('spotify') + def _emit(event_type: str, **payload): + if progress_callback: + try: + progress_callback(event_type, payload) + except Exception: + logger.debug("Watchlist scan progress callback failed for %s", event_type, exc_info=True) + + _emit('scan_started', profile_id=profile_id, total_artists=len(watchlist_artists)) + + # Keep this as a plain source list; resolve the client right before each use. + providers_to_backfill = [ + source for source in self._watchlist_source_priority() + if source in {'spotify', 'itunes', 'deezer', 'discogs'} + ] + + for provider in providers_to_backfill: try: - from config.settings import config_manager as _cfg - if _cfg.get('discogs.token', ''): - providers_to_backfill.append('discogs') - except Exception: - pass + logger.info("Checking for missing %s IDs in watchlist...", provider) + self._backfill_missing_ids(watchlist_artists, provider) + except Exception as backfill_error: + logger.warning("Error during %s ID backfilling: %s", provider, backfill_error) - for provider in providers_to_backfill: - try: - self._backfill_missing_ids(all_watchlist_artists, provider) - except Exception as backfill_error: - logger.warning(f"Error during {provider} ID backfilling: {backfill_error}") - # Continue with scan even if backfilling fails - - scan_results = [] - for i, artist in enumerate(watchlist_artists): - if self.spotify_client and self.spotify_client.is_rate_limited(): - self._disable_spotify_for_run("global Spotify rate limit active") + lookback_period = self._get_lookback_period_setting() + is_full_discography = (lookback_period == 'all') + artist_count = len(watchlist_artists) - try: - result = self.scan_artist(artist) - scan_results.append(result) - if self.spotify_client and self.spotify_client.is_rate_limited(): - self._disable_spotify_for_run("global Spotify rate limit active") + base_artist_delay = DELAY_BETWEEN_ARTISTS + base_album_delay = DELAY_BETWEEN_ALBUMS + if is_full_discography: + base_artist_delay *= 2.0 + base_album_delay *= 2.0 + if artist_count > 200: + base_artist_delay *= 1.5 + base_album_delay *= 1.25 + elif artist_count > 100: + base_artist_delay *= 1.25 - if result.success: - logger.info(f"Scanned {artist.artist_name}: {result.new_tracks_found} new tracks found") - else: - logger.warning(f"Failed to scan {artist.artist_name}: {result.error_message}") + artist_delay = base_artist_delay + album_delay = base_album_delay + logger.info( + "Scan parameters: %s artists, lookback=%s, delays: %.1fs/artist, %.1fs/album", + artist_count, + lookback_period, + artist_delay, + album_delay, + ) - # Rate limiting: Add delay between artists to avoid hitting Spotify API limits - # This is critical to prevent getting banned for 6+ hours - if i < len(watchlist_artists) - 1: # Don't delay after the last artist - logger.debug(f"Rate limiting: waiting {DELAY_BETWEEN_ARTISTS}s before scanning next artist") - time.sleep(DELAY_BETWEEN_ARTISTS) - - except Exception as e: - logger.error(f"Error scanning artist {artist.artist_name}: {e}") + for i, artist in enumerate(watchlist_artists): + if cancel_check and cancel_check(): + logger.info("Watchlist scan cancelled after %s/%s artists", i, len(watchlist_artists)) + if scan_state is not None: + successful_scans = [r for r in scan_results if r.success] + scan_state['status'] = 'cancelled' + scan_state['current_phase'] = 'cancelled' + scan_state['summary'] = { + 'total_artists': i, + 'successful_scans': len(successful_scans), + 'new_tracks_found': sum(r.new_tracks_found for r in successful_scans), + 'tracks_added_to_wishlist': sum(r.tracks_added_to_wishlist for r in successful_scans), + 'cancelled': True, + } + _emit('cancelled', processed=i, total=len(watchlist_artists)) + break + + source_artist_id = ( + artist.spotify_artist_id + or artist.itunes_artist_id + or artist.deezer_artist_id + or artist.discogs_artist_id + or str(artist.id) + ) + + try: + discography_result = self.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp) + if discography_result is None: scan_results.append(ScanResult( artist_name=artist.artist_name, - spotify_artist_id=artist.spotify_artist_id, + spotify_artist_id=source_artist_id, albums_checked=0, new_tracks_found=0, tracks_added_to_wishlist=0, success=False, - error_message=str(e) + error_message="Failed to get artist discography", )) - - # Log summary + _emit( + 'artist_error', + artist_name=artist.artist_name, + profile_id=profile_id, + error_message="Failed to get artist discography", + ) + continue + + if isinstance(discography_result, list): + albums = discography_result + artist_image_url = self.get_artist_image_url(artist) or '' + album_fetcher = lambda album_id, album_name='': self.metadata_service.get_album(album_id) + else: + source = discography_result.source + albums = discography_result.albums + source_artist_id = discography_result.artist_id + artist_image_url = discography_result.image_url or self.get_artist_image_url(artist) or '' + album_fetcher = lambda album_id, album_name='': self._get_album_data_for_source(source, album_id, album_name) + + absolute_index = artist_index_offset + i + 1 + if scan_state is not None: + scan_state.update({ + 'current_artist_index': absolute_index, + 'current_artist_name': artist.artist_name, + 'current_artist_image_url': artist_image_url, + 'current_phase': 'fetching_discography', + 'albums_to_check': 0, + 'albums_checked': 0, + 'current_album': '', + 'current_album_image_url': '', + 'current_track_name': '', + }) + + _emit( + 'artist_started', + artist_name=artist.artist_name, + artist_index=absolute_index, + total_artists=total_artists_override if total_artists_override is not None else len(watchlist_artists), + profile_id=profile_id, + artist_image_url=artist_image_url, + ) + + if scan_state is not None: + scan_state.update({ + 'current_phase': 'checking_albums', + 'albums_to_check': len(albums), + 'albums_checked': 0, + }) + + artist_new_tracks = 0 + artist_added_tracks = 0 + + for album_index, album in enumerate(albums): + try: + album_data = album_fetcher(album.id, getattr(album, 'name', '')) + tracks = self._extract_track_items(album_data) + if not album_data or not tracks: + logger.debug("Skipping album %s (id=%s): no track data returned", album.name, album.id) + continue + + album_name = getattr(album, 'name', '') + if isinstance(album_data, dict): + album_name = album_data.get('name', album_name) + else: + album_name = getattr(album_data, 'name', album_name) + + if self._has_placeholder_tracks(tracks): + logger.info("Skipping album with placeholder tracks: %s", album_name) + continue + if not self._should_include_release(len(tracks), artist): + continue + + album_image_url = '' + album_images = [] + if isinstance(album_data, dict): + album_images = album_data.get('images') or [] + else: + album_images = getattr(album_data, 'images', None) or [] + if album_images: + first_image = album_images[0] + if isinstance(first_image, dict): + album_image_url = first_image.get('url', '') + + if scan_state is not None: + scan_state.update({ + 'albums_checked': album_index + 1, + 'current_album': album_name, + 'current_album_image_url': album_image_url, + 'current_phase': f'checking_album_{album_index + 1}_of_{len(albums)}', + }) + + _emit( + 'album_started', + artist_name=artist.artist_name, + album_name=album_name, + album_index=album_index + 1, + total_albums=len(albums), + album_image_url=album_image_url, + ) + + for track in tracks: + if not self._should_include_track(track, album_data, artist): + continue + + track_name = track.get('name', 'Unknown Track') + if scan_state is not None: + scan_state['current_track_name'] = track_name + + if self.is_track_missing_from_library(track, album_name=album_name): + artist_new_tracks += 1 + if scan_state is not None: + scan_state['tracks_found_this_scan'] += 1 + + if self.add_track_to_wishlist(track, album_data, artist): + artist_added_tracks += 1 + if scan_state is not None: + scan_state['tracks_added_this_scan'] += 1 + + track_artists = track.get('artists', []) + track_artist_name = track_artists[0].get('name', 'Unknown Artist') if track_artists else 'Unknown Artist' + if scan_state is not None: + scan_state['recent_wishlist_additions'].insert(0, { + 'track_name': track_name, + 'artist_name': track_artist_name, + 'album_image_url': album_image_url, + }) + if len(scan_state['recent_wishlist_additions']) > 10: + scan_state['recent_wishlist_additions'].pop() + + if album_index < len(albums) - 1: + time.sleep(album_delay) + + except Exception as e: + logger.warning("Error checking album %s: %s", album.name, e) + continue + + self.update_artist_scan_timestamp(artist) + + scan_results.append(ScanResult( + artist_name=artist.artist_name, + spotify_artist_id=source_artist_id or artist.spotify_artist_id or '', + albums_checked=len(albums), + new_tracks_found=artist_new_tracks, + tracks_added_to_wishlist=artist_added_tracks, + success=True, + )) + + _emit( + 'artist_completed', + artist_name=artist.artist_name, + artist_index=absolute_index, + total_artists=total_artists_override if total_artists_override is not None else len(watchlist_artists), + profile_id=profile_id, + albums_checked=len(albums), + new_tracks_found=artist_new_tracks, + tracks_added_to_wishlist=artist_added_tracks, + ) + + try: + if scan_state is not None: + scan_state['current_phase'] = 'fetching_similar_artists' + artist_profile_id = getattr(artist, 'profile_id', profile_id) + spotify_authenticated = self.spotify_client and self.spotify_client.is_spotify_authenticated() + if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=artist_profile_id): + logger.info("Similar artists for %s are cached and fresh (profile %s)", artist.artist_name, artist_profile_id) + self._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=artist_profile_id) + else: + logger.info("Fetching similar artists for %s (profile %s)...", artist.artist_name, artist_profile_id) + self.update_similar_artists(artist, profile_id=artist_profile_id, source_artist_id=source_artist_id) + logger.info("Similar artists updated for %s", artist.artist_name) + except Exception as similar_error: + logger.warning("Failed to update similar artists for %s: %s", artist.artist_name, similar_error) + + if i < len(watchlist_artists) - 1: + if scan_state is not None: + scan_state['current_phase'] = 'rate_limiting' + time.sleep(artist_delay) + + except Exception as e: + logger.error("Error scanning artist %s: %s", artist.artist_name, e) + scan_results.append(ScanResult( + artist_name=artist.artist_name, + spotify_artist_id=source_artist_id, + albums_checked=0, + new_tracks_found=0, + tracks_added_to_wishlist=0, + success=False, + error_message=str(e), + )) + _emit( + 'artist_error', + artist_name=artist.artist_name, + artist_index=artist_index_offset + i + 1, + total_artists=total_artists_override if total_artists_override is not None else len(watchlist_artists), + profile_id=profile_id, + error_message=str(e), + ) + + if scan_state is not None: successful_scans = [r for r in scan_results if r.success] total_new_tracks = sum(r.new_tracks_found for r in successful_scans) total_added_to_wishlist = sum(r.tracks_added_to_wishlist for r in successful_scans) - - logger.info(f"Watchlist scan complete: {len(successful_scans)}/{len(scan_results)} artists scanned successfully") - logger.info(f"Found {total_new_tracks} new tracks, added {total_added_to_wishlist} to wishlist") + scan_state['results'] = list(scan_state.get('results', [])) + scan_results + if scan_state.get('status') != 'cancelled': + scan_state['status'] = 'completed' + scan_state['completed_at'] = datetime.now() + scan_state['current_phase'] = 'completed' + scan_state['summary'] = { + 'total_artists': len(scan_results), + 'successful_scans': len(successful_scans), + 'new_tracks_found': total_new_tracks, + 'tracks_added_to_wishlist': total_added_to_wishlist, + } - # Populate discovery pool with tracks from similar artists - logger.info("Starting discovery pool population...") - if self.spotify_client and self.spotify_client.is_rate_limited(): - self._disable_spotify_for_run("global Spotify rate limit active") - self.populate_discovery_pool() - - # Populate seasonal content (runs independently with its own threshold) - logger.info("Updating seasonal content...") - self._populate_seasonal_content() - - # Generate Last.fm Radio playlists for top tracks (max once per week) - self._generate_lastfm_radio_playlists() - - # Sync Spotify library cache (runs after main scan) - try: - if self.spotify_client and self.spotify_client.is_rate_limited(): - self._disable_spotify_for_run("global Spotify rate limit active") - self.sync_spotify_library_cache() - except Exception as lib_err: - logger.warning(f"Error syncing Spotify library cache: {lib_err}") - - return scan_results - - except Exception as e: - logger.error(f"Error during watchlist scan: {e}") - return [] - finally: - self._reset_spotify_run_state() + _emit( + 'scan_completed', + profile_id=profile_id, + total_artists=len(watchlist_artists), + total_scanned=len(scan_results), + successful_scans=len([r for r in scan_results if r.success]), + new_tracks_found=sum(r.new_tracks_found for r in scan_results if r.success), + tracks_added_to_wishlist=sum(r.tracks_added_to_wishlist for r in scan_results if r.success), + ) + return scan_results - def scan_artist(self, watchlist_artist: WatchlistArtist) -> ScanResult: - """ - Scan a single artist for new releases. - Only checks releases after the last scan timestamp. - Uses the active provider (Spotify if authenticated, otherwise iTunes). - """ - try: - logger.info(f"Scanning artist: {watchlist_artist.artist_name}") - - # Get the active client and artist ID based on provider - client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist) - - if client is None or artist_id is None: - return ScanResult( - artist_name=watchlist_artist.artist_name, - spotify_artist_id=watchlist_artist.spotify_artist_id or '', - albums_checked=0, - new_tracks_found=0, - tracks_added_to_wishlist=0, - success=False, - error_message=f"No {self.metadata_service.get_active_provider()} ID available for this artist" - ) - - logger.info(f"Using {provider} provider for {watchlist_artist.artist_name} (ID: {artist_id})") - - # Update artist image if missing or on every scan to keep fresh - try: - image_url = None - artist_data = client.get_artist(artist_id) - if artist_data: - if 'images' in artist_data and artist_data['images']: - # Spotify/Deezer format: array of {url, height, width} - image_url = artist_data['images'][1]['url'] if len(artist_data['images']) > 1 else artist_data['images'][0]['url'] - elif artist_data.get('image_url'): - # Direct image_url format (iTunes/some providers) - image_url = artist_data['image_url'] - - if image_url: - db_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or watchlist_artist.deezer_artist_id or artist_id - self.database.update_watchlist_artist_image(db_artist_id, image_url) - if not watchlist_artist.image_url: - logger.info(f"Backfilled artist image for {watchlist_artist.artist_name}") - else: - logger.debug(f"No image available for {watchlist_artist.artist_name} from {provider}") - except Exception as img_error: - logger.warning(f"Could not update artist image for {watchlist_artist.artist_name}: {img_error}") - - # Get artist discography using active provider - albums = self._get_artist_discography_with_client(client, artist_id, watchlist_artist.last_scan_timestamp, lookback_days=watchlist_artist.lookback_days) - - if albums is None: - return ScanResult( - artist_name=watchlist_artist.artist_name, - spotify_artist_id=watchlist_artist.spotify_artist_id or '', - albums_checked=0, - new_tracks_found=0, - tracks_added_to_wishlist=0, - success=False, - error_message=f"Failed to get artist discography from {provider}" - ) - - logger.info(f"Found {len(albums)} albums/singles to check for {watchlist_artist.artist_name}") - - # Safety check: Limit number of albums to scan to prevent extremely long sessions - MAX_ALBUMS_PER_ARTIST = 50 # Reasonable limit to prevent API abuse - if len(albums) > MAX_ALBUMS_PER_ARTIST: - logger.warning(f"Artist {watchlist_artist.artist_name} has {len(albums)} albums, limiting to {MAX_ALBUMS_PER_ARTIST} most recent") - albums = albums[:MAX_ALBUMS_PER_ARTIST] # Most recent albums are first - - # Check each album/single for missing tracks - new_tracks_found = 0 - tracks_added_to_wishlist = 0 - - for album_index, album in enumerate(albums): - try: - # Get full album data - logger.info(f"Checking album {album_index + 1}/{len(albums)}: {album.name}") - album_data = client.get_album(album.id) - if not album_data: - continue - - # Get album tracks (works for both Spotify and iTunes) - # Spotify's get_album() includes tracks, but we use get_album_tracks() for consistency - tracks_data = client.get_album_tracks(album.id) - if not tracks_data or not tracks_data.get('items'): - continue - - tracks = tracks_data['items'] - logger.debug(f"Checking album: {album_data.get('name', 'Unknown')} ({len(tracks)} tracks)") - - # Check if user wants this type of release - if not self._should_include_release(len(tracks), watchlist_artist): - release_type = "album" if len(tracks) >= 7 else ("EP" if len(tracks) >= 4 else "single") - logger.debug(f"Skipping {release_type}: {album_data.get('name', 'Unknown')} - user preference") - continue - - # Skip albums with placeholder track names (unreleased tracklist) - # Spotify uses "Track 1", "Track 2", etc. for unannounced tracks - if self._has_placeholder_tracks(tracks): - logger.info(f"Skipping album with placeholder tracks (unreleased tracklist): {album_data.get('name', 'Unknown')}") - continue - - # Check each track - for track in tracks: - # Check content type filters (live, remix, acoustic, compilation) - if not self._should_include_track(track, album_data, watchlist_artist): - continue # Skip this track based on content type preferences - - if self.is_track_missing_from_library(track, album_name=album_data.get('name')): - new_tracks_found += 1 - - # Add to wishlist - if self.add_track_to_wishlist(track, album_data, watchlist_artist): - tracks_added_to_wishlist += 1 - - # Rate limiting: Add delay between albums to prevent API abuse - # This is especially important for artists with many albums - if album_index < len(albums) - 1: # Don't delay after the last album - logger.debug(f"Rate limiting: waiting {DELAY_BETWEEN_ALBUMS}s before next album") - time.sleep(DELAY_BETWEEN_ALBUMS) - - except Exception as e: - logger.warning(f"Error checking album {album.name}: {e}") - continue - - # Update last scan timestamp for this artist - self.update_artist_scan_timestamp(watchlist_artist) - - # Fetch and store similar artists for discovery feature (with caching to avoid over-polling) - # Similar artists are fetched from MusicMap (works with any source) and matched to both Spotify and iTunes - source_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or str(watchlist_artist.id) - try: - # Check if we have fresh similar artists cached (< 30 days old) - # If Spotify is authenticated, also require Spotify IDs to be present - spotify_authenticated = self.spotify_client and self.spotify_client.is_spotify_authenticated() - artist_profile_id = getattr(watchlist_artist, 'profile_id', 1) - if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=artist_profile_id): - logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping MusicMap fetch") - # Even if cached, backfill missing iTunes IDs (seamless dual-source support) - self._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=artist_profile_id) - else: - logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}...") - self.update_similar_artists(watchlist_artist, profile_id=artist_profile_id) - logger.info(f"Similar artists updated for {watchlist_artist.artist_name}") - except Exception as similar_error: - logger.warning(f"Failed to update similar artists for {watchlist_artist.artist_name}: {similar_error}") - - return ScanResult( - artist_name=watchlist_artist.artist_name, - spotify_artist_id=watchlist_artist.spotify_artist_id or '', - albums_checked=len(albums), - new_tracks_found=new_tracks_found, - tracks_added_to_wishlist=tracks_added_to_wishlist, - success=True - ) - - except Exception as e: - logger.error(f"Error scanning artist {watchlist_artist.artist_name}: {e}") - return ScanResult( - artist_name=watchlist_artist.artist_name, - spotify_artist_id=watchlist_artist.spotify_artist_id or '', - albums_checked=0, - new_tracks_found=0, - tracks_added_to_wishlist=0, - success=False, - error_message=str(e) - ) - - def get_artist_discography(self, spotify_artist_id: str, last_scan_timestamp: Optional[datetime] = None) -> Optional[List]: + def get_artist_discography( + self, + spotify_artist_id: str, + last_scan_timestamp: Optional[datetime] = None, + lookback_days: Optional[int] = None, + ) -> Optional[List]: """ Get artist's discography from Spotify, optionally filtered by release date. @@ -910,61 +1225,16 @@ class WatchlistScanner: spotify_artist_id: Spotify artist ID last_scan_timestamp: Only return releases after this date (for incremental scans) If None, uses lookback period setting from database + lookback_days: Optional per-artist override for lookback period """ try: - # Determine if we need the full discography or just recent releases. - # Spotify returns albums sorted newest-first, so for time-bounded scans - # we only need the first page (50 albums) — this cuts API calls by ~90% - # for prolific artists (262 albums = 27 calls → 1 call). - needs_full_discog = False - cutoff_timestamp = last_scan_timestamp - - if cutoff_timestamp is None: - if lookback_days is not None: - cutoff_timestamp = datetime.now(timezone.utc) - timedelta(days=lookback_days) - logger.info(f"Using per-artist lookback: {lookback_days} days (cutoff: {cutoff_timestamp})") - else: - lookback_period = self._get_lookback_period_setting() - if lookback_period == 'all': - needs_full_discog = True - else: - days = int(lookback_period) - cutoff_timestamp = datetime.now(timezone.utc) - timedelta(days=days) - logger.info(f"Using global lookback period: {lookback_period} days (cutoff: {cutoff_timestamp})") - - # Fetch albums — limit pagination unless full discography is needed - logger.debug(f"Fetching discography for artist {spotify_artist_id}" + - (" (full)" if needs_full_discog else " (recent only, max 1 page)")) - albums = self.spotify_client.get_artist_albums( - spotify_artist_id, album_type='album,single', limit=50, - skip_cache=True, max_pages=0 if needs_full_discog else 1 + return self._get_artist_discography_with_client( + self.spotify_client, + spotify_artist_id, + last_scan_timestamp, + lookback_days=lookback_days, ) - if not albums: - logger.warning(f"No albums found for artist {spotify_artist_id}") - return [] - - # Add small delay after fetching artist discography to be extra safe - time.sleep(0.3) # 300ms breathing room - - # Filter by release date if we have a cutoff timestamp - if cutoff_timestamp: - filtered_albums = [] - for album in albums: - if self.is_album_after_timestamp(album, cutoff_timestamp): - filtered_albums.append(album) - - logger.info(f"Filtered {len(albums)} albums to {len(filtered_albums)} released after {cutoff_timestamp}") - albums = filtered_albums - - # Skip future/unreleased albums — no real audio available yet - now = datetime.now(timezone.utc) - released = [a for a in albums if not self._is_future_release(a, now)] - skipped = len(albums) - len(released) - if skipped: - logger.info(f"Skipped {skipped} future/unreleased albums (will be picked up after release)") - return released - except Exception as e: logger.error(f"Error getting discography for artist {spotify_artist_id}: {e}") return None @@ -1098,8 +1368,8 @@ class WatchlistScanner: update_fn = { 'spotify': self.database.update_watchlist_spotify_id, 'itunes': self.database.update_watchlist_itunes_id, - 'deezer': getattr(self.database, 'update_watchlist_deezer_id', None), - 'discogs': getattr(self.database, 'update_watchlist_discogs_id', None), + 'deezer': self.database.update_watchlist_deezer_id, + 'discogs': self.database.update_watchlist_discogs_id, }.get(provider) if not match_fn or not update_fn: @@ -1203,10 +1473,11 @@ class WatchlistScanner: def _match_to_spotify(self, artist_name: str) -> Optional[str]: """Match artist name to Spotify ID using fuzzy name comparison.""" try: - if hasattr(self, '_metadata_service') and self._metadata_service: - results = self._metadata_service.spotify.search_artists(artist_name, limit=5) - else: - results = self.spotify_client.search_artists(artist_name, limit=5) + client = get_client_for_source('spotify') + if not client: + return None + + results = client.search_artists(artist_name, limit=5, allow_fallback=False) return self._best_artist_match(results, artist_name) except Exception as e: @@ -1786,7 +2057,7 @@ class WatchlistScanner: try: # Try Spotify search (only when Spotify is the configured primary source) if self._spotify_is_primary_source(): - searched_results = self.spotify_client.search_artists(artist_name, limit=1) + searched_results = self.spotify_client.search_artists(artist_name, limit=1, allow_fallback=False) if searched_results and len(searched_results) > 0: searched_spotify_id = searched_results[0].id except Exception as e: @@ -1824,7 +2095,11 @@ class WatchlistScanner: # Try to match on Spotify (only when Spotify is the configured primary source) if self._spotify_is_primary_source(): try: - spotify_results = self.spotify_client.search_artists(artist_name_to_match, limit=1) + spotify_results = self.spotify_client.search_artists( + artist_name_to_match, + limit=1, + allow_fallback=False, + ) if spotify_results and len(spotify_results) > 0: spotify_artist = spotify_results[0] # Skip if this is the searched artist @@ -1939,7 +2214,13 @@ class WatchlistScanner: logger.error(f"Error backfilling similar artists {fallback_source if 'fallback_source' in dir() else 'fallback'} IDs: {e}") return 0 - def update_similar_artists(self, watchlist_artist: WatchlistArtist, limit: int = 10, profile_id: int = 1) -> bool: + def update_similar_artists( + self, + watchlist_artist: WatchlistArtist, + limit: int = 10, + profile_id: int = 1, + source_artist_id: Optional[str] = None, + ) -> bool: """ Fetch and store similar artists for a watchlist artist. Called after each artist scan to build discovery pool. @@ -1957,8 +2238,15 @@ class WatchlistScanner: logger.info(f"Found {len(similar_artists)} similar artists for {watchlist_artist.artist_name}") - # Use consistent source artist ID (prefer Spotify, fall back to iTunes or internal ID) - source_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or str(watchlist_artist.id) + # Use the ID that matched the scan source when available; otherwise fall back to any known ID. + source_artist_id = ( + source_artist_id + or watchlist_artist.spotify_artist_id + or watchlist_artist.itunes_artist_id + or watchlist_artist.deezer_artist_id + or watchlist_artist.discogs_artist_id + or str(watchlist_artist.id) + ) # Store each similar artist in database stored_count = 0 @@ -2115,7 +2403,7 @@ class WatchlistScanner: artist_id, album_type='album,single,ep', limit=50, - skip_cache=True + skip_cache=True, ) else: # itunes or deezer fallback all_albums = itunes_client.get_artist_albums( @@ -2340,7 +2628,11 @@ class WatchlistScanner: # Try Spotify first if available if spotify_available: try: - search_results = self.spotify_client.search_albums(f"album:{album_row['title']} artist:{album_row['artist_name']}", limit=1) + search_results = self.spotify_client.search_albums( + f"album:{album_row['title']} artist:{album_row['artist_name']}", + limit=1, + allow_fallback=False, + ) if search_results and len(search_results) > 0: spotify_album = search_results[0] album_data = self.spotify_client.get_album(spotify_album.id) @@ -2526,7 +2818,7 @@ class WatchlistScanner: artist.spotify_artist_id, album_type='album,single,ep', limit=5, - skip_cache=True + skip_cache=True, ) if not recent_releases: @@ -2767,7 +3059,7 @@ class WatchlistScanner: artist.spotify_artist_id, album_type='album,single,ep', limit=20, - skip_cache=True + skip_cache=True, ) for album in albums or []: process_album(album, artist.artist_name, artist.spotify_artist_id, fallback_id if fallback_source == 'itunes' else None, 'spotify') @@ -2824,7 +3116,7 @@ class WatchlistScanner: artist.similar_artist_spotify_id, album_type='album,single,ep', limit=20, - skip_cache=True + skip_cache=True, ) for album in albums or []: process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, fallback_id if fallback_source == 'itunes' else None, 'spotify') diff --git a/tests/test_watchlist_scanner_scan.py b/tests/test_watchlist_scanner_scan.py new file mode 100644 index 00000000..57d8909d --- /dev/null +++ b/tests/test_watchlist_scanner_scan.py @@ -0,0 +1,401 @@ +import sys +import types + + +if "spotipy" not in sys.modules: + spotipy = types.ModuleType("spotipy") + + class _DummySpotify: + def __init__(self, *args, **kwargs): + pass + + oauth2 = types.ModuleType("spotipy.oauth2") + + class _DummyOAuth: + def __init__(self, *args, **kwargs): + pass + + spotipy.Spotify = _DummySpotify + oauth2.SpotifyOAuth = _DummyOAuth + oauth2.SpotifyClientCredentials = _DummyOAuth + spotipy.oauth2 = oauth2 + sys.modules["spotipy"] = spotipy + sys.modules["spotipy.oauth2"] = oauth2 + +if "config.settings" not in sys.modules: + config_pkg = types.ModuleType("config") + settings_mod = types.ModuleType("config.settings") + + class _DummyConfigManager: + def get(self, key, default=None): + return default + + def get_active_media_server(self): + return "plex" + + settings_mod.config_manager = _DummyConfigManager() + config_pkg.settings = settings_mod + sys.modules["config"] = config_pkg + sys.modules["config.settings"] = settings_mod + +if "core.matching_engine" not in sys.modules: + matching_engine_mod = types.ModuleType("core.matching_engine") + + class _DummyMatchingEngine: + def clean_title(self, title): + return title + + matching_engine_mod.MusicMatchingEngine = _DummyMatchingEngine + sys.modules["core.matching_engine"] = matching_engine_mod + +import core.watchlist_scanner as watchlist_scanner_module +from core.watchlist_scanner import WatchlistScanner + + +class _FakeSpotifyClient: + def __init__(self, search_results=None): + self.search_results = list(search_results or []) + self.search_calls = [] + + def is_spotify_authenticated(self): + return False + + def search_artists(self, query, limit=1, allow_fallback=True): + self.search_calls.append((query, limit, allow_fallback)) + return list(self.search_results) if allow_fallback else [] + + +class _FakeMetadataService: + def __init__(self, album_data, spotify_client=None): + self.spotify = spotify_client or _FakeSpotifyClient() + self.itunes = types.SimpleNamespace() + self._album_data = album_data + + def get_album(self, album_id): + return self._album_data + + +class _FakeSourceClient: + def __init__(self, *, artist_id: str, albums, image_url: str): + self.artist_id = artist_id + self.albums = list(albums) + self.image_url = image_url + self.search_calls = [] + self.album_calls = [] + self.artist_calls = [] + + def search_artists(self, query, limit=1, **kwargs): + self.search_calls.append((query, limit, kwargs)) + return [types.SimpleNamespace(id=self.artist_id, name=query)] + + def get_artist_albums(self, artist_id, album_type='album,single', limit=50, **kwargs): + self.album_calls.append((artist_id, album_type, limit, kwargs)) + return list(self.albums) + + def get_artist(self, artist_id): + self.artist_calls.append(artist_id) + return { + "id": artist_id, + "images": [{"url": self.image_url}] if self.image_url else [], + } + + +class _FakeDB: + def __init__(self, artists): + self.artists = artists + self.similar_calls = [] + + def get_watchlist_artists(self, profile_id=None): + return list(self.artists) + + def has_fresh_similar_artists(self, *args, **kwargs): + self.similar_calls.append((args, kwargs)) + return False + + +def _build_artist(name="Artist One", profile_id=11): + return types.SimpleNamespace( + artist_name=name, + spotify_artist_id="sp-artist", + itunes_artist_id="it-artist", + deezer_artist_id="dz-artist", + discogs_artist_id="dg-artist", + last_scan_timestamp=None, + id=123, + profile_id=profile_id, + include_albums=True, + include_eps=True, + include_singles=True, + include_live=False, + include_remixes=False, + include_acoustic=False, + include_compilations=False, + include_instrumentals=False, + lookback_days=7, + image_url=None, + ) + + +def _build_scanner(album_data, artists): + scanner = WatchlistScanner(metadata_service=_FakeMetadataService(album_data)) + scanner._database = _FakeDB(artists) + scanner._wishlist_service = types.SimpleNamespace() + scanner._matching_engine = types.SimpleNamespace() + return scanner + + +def test_scan_watchlist_profile_loads_artists_and_applies_overrides(monkeypatch): + artist = _build_artist() + scanner = _build_scanner({"tracks": {"items": []}}, [artist]) + + loaded_profiles = [] + override_calls = [] + scan_calls = [] + + monkeypatch.setattr(scanner.database, "get_watchlist_artists", lambda profile_id=None: loaded_profiles.append(profile_id) or [artist]) + monkeypatch.setattr(scanner, "_apply_global_watchlist_overrides", lambda artists: override_calls.append(list(artists))) + monkeypatch.setattr(scanner, "scan_watchlist_artists", lambda artists, **kwargs: scan_calls.append((list(artists), kwargs)) or ["ok"]) + + result = scanner.scan_watchlist_profile(42) + + assert result == ["ok"] + assert loaded_profiles == [42] + assert override_calls and override_calls[0][0].artist_name == "Artist One" + assert scan_calls and scan_calls[0][0][0].artist_name == "Artist One" + assert scan_calls[0][1]["profile_id"] == 42 + + +def test_scan_watchlist_artists_scans_tracks_and_updates_state(monkeypatch): + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0) + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0) + + artist = _build_artist() + album = types.SimpleNamespace(id="album-1", name="Album One") + album_data = { + "name": "Album One", + "images": [{"url": "https://example.com/album.jpg"}], + "tracks": { + "items": [ + { + "id": "track-1", + "name": "Track One", + "track_number": 1, + "disc_number": 1, + "artists": [{"name": "Artist One"}], + } + ] + }, + } + scanner = _build_scanner(album_data, [artist]) + scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False + + monkeypatch.setattr(scanner, "_backfill_missing_ids", lambda *args, **kwargs: None) + monkeypatch.setattr(scanner, "get_artist_image_url", lambda *_args, **_kwargs: "https://example.com/artist.jpg") + monkeypatch.setattr(scanner, "get_artist_discography_for_watchlist", lambda *_args, **_kwargs: [album]) + monkeypatch.setattr(scanner, "_get_lookback_period_setting", lambda: "30") + monkeypatch.setattr(scanner, "_get_rescan_cutoff", lambda: None) + monkeypatch.setattr(scanner, "_should_include_release", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_should_include_track", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "is_track_missing_from_library", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0) + + scan_state = {} + results = scanner.scan_watchlist_artists([artist], scan_state=scan_state) + + assert len(results) == 1 + assert results[0].success is True + assert results[0].new_tracks_found == 1 + assert results[0].tracks_added_to_wishlist == 1 + assert scan_state["status"] == "completed" + assert scan_state["summary"]["successful_scans"] == 1 + assert scan_state["summary"]["new_tracks_found"] == 1 + assert scan_state["summary"]["tracks_added_to_wishlist"] == 1 + assert scan_state["recent_wishlist_additions"][0]["track_name"] == "Track One" + + +def test_scan_watchlist_artists_skips_placeholder_tracklists(monkeypatch): + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0) + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0) + + artist = _build_artist() + album = types.SimpleNamespace(id="album-1", name="Album One") + album_data = { + "name": "Album One", + "images": [{"url": "https://example.com/album.jpg"}], + "tracks": { + "items": [ + { + "id": "track-1", + "name": "Track 1", + "track_number": 1, + "disc_number": 1, + "artists": [{"name": "Artist One"}], + }, + { + "id": "track-2", + "name": "Track 2", + "track_number": 2, + "disc_number": 1, + "artists": [{"name": "Artist One"}], + }, + ] + }, + } + scanner = _build_scanner(album_data, [artist]) + scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False + + monkeypatch.setattr(scanner, "_backfill_missing_ids", lambda *args, **kwargs: None) + monkeypatch.setattr(scanner, "get_artist_image_url", lambda *_args, **_kwargs: "https://example.com/artist.jpg") + monkeypatch.setattr(scanner, "get_artist_discography_for_watchlist", lambda *_args, **_kwargs: [album]) + monkeypatch.setattr(scanner, "_get_lookback_period_setting", lambda: "30") + monkeypatch.setattr(scanner, "_get_rescan_cutoff", lambda: None) + monkeypatch.setattr(scanner, "_should_include_release", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_should_include_track", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "is_track_missing_from_library", lambda *_args, **_kwargs: True) + + add_calls = [] + monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *args, **kwargs: add_calls.append((args, kwargs)) or True) + monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0) + + scan_state = {} + results = scanner.scan_watchlist_artists([artist], scan_state=scan_state) + + assert len(results) == 1 + assert results[0].success is True + assert results[0].new_tracks_found == 0 + assert results[0].tracks_added_to_wishlist == 0 + assert add_calls == [] + assert scan_state["summary"]["new_tracks_found"] == 0 + assert scan_state["summary"]["tracks_added_to_wishlist"] == 0 + + +def test_scan_watchlist_artists_honors_cancel_check(monkeypatch): + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0) + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0) + + artist_a = _build_artist("Artist One") + artist_b = _build_artist("Artist Two") + album = types.SimpleNamespace(id="album-1", name="Album One") + album_data = { + "name": "Album One", + "tracks": {"items": []}, + } + scanner = _build_scanner(album_data, [artist_a, artist_b]) + scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False + + monkeypatch.setattr(scanner, "_backfill_missing_ids", lambda *args, **kwargs: None) + monkeypatch.setattr(scanner, "get_artist_image_url", lambda *_args, **_kwargs: "https://example.com/artist.jpg") + monkeypatch.setattr(scanner, "get_artist_discography_for_watchlist", lambda *_args, **_kwargs: [album]) + monkeypatch.setattr(scanner, "_get_lookback_period_setting", lambda: "30") + monkeypatch.setattr(scanner, "_get_rescan_cutoff", lambda: None) + monkeypatch.setattr(scanner, "_should_include_release", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_should_include_track", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "is_track_missing_from_library", lambda *_args, **_kwargs: False) + monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *_args, **_kwargs: False) + monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0) + + cancels = iter([False, True]) + scan_state = {} + results = scanner.scan_watchlist_artists( + [artist_a, artist_b], + scan_state=scan_state, + cancel_check=lambda: next(cancels), + ) + + assert len(results) == 1 + assert results[0].artist_name == "Artist One" + assert scan_state["status"] == "cancelled" + assert scan_state["summary"]["cancelled"] is True + assert scan_state["summary"]["successful_scans"] == 1 + + +def test_get_artist_discography_for_watchlist_prefers_primary_source(monkeypatch): + monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None)) + monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + + deezer_album = types.SimpleNamespace(id="dz-album", name="Deezer Album", release_date=None) + spotify_album = types.SimpleNamespace(id="sp-album", name="Spotify Album", release_date=None) + + deezer_client = _FakeSourceClient(artist_id="dz-artist", albums=[deezer_album], image_url="https://example.com/deezer.jpg") + spotify_client = _FakeSourceClient(artist_id="sp-artist", albums=[spotify_album], image_url="https://example.com/spotify.jpg") + + def fake_get_client_for_source(source): + return {"deezer": deezer_client, "spotify": spotify_client}.get(source) + + monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source) + + artist = _build_artist() + artist.spotify_artist_id = "sp-artist" + artist.deezer_artist_id = "dz-artist" + + scanner = _build_scanner({"tracks": {"items": []}}, [artist]) + scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False + scanner._get_lookback_period_setting = lambda: "30" + scanner._get_rescan_cutoff = lambda: None + + result = scanner.get_artist_discography_for_watchlist(artist, None) + + assert result is not None + assert result.source == "deezer" + assert result.artist_id == "dz-artist" + assert result.albums and result.albums[0].id == "dz-album" + assert deezer_client.album_calls + assert spotify_client.album_calls == [] + + +def test_get_artist_discography_for_watchlist_falls_back_when_primary_empty(monkeypatch): + monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None)) + monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + + deezer_client = _FakeSourceClient(artist_id="dz-artist", albums=[], image_url="https://example.com/deezer.jpg") + spotify_album = types.SimpleNamespace(id="sp-album", name="Spotify Album", release_date=None) + spotify_client = _FakeSourceClient(artist_id="sp-artist", albums=[spotify_album], image_url="https://example.com/spotify.jpg") + + def fake_get_client_for_source(source): + return {"deezer": deezer_client, "spotify": spotify_client}.get(source) + + monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source) + + artist = _build_artist() + artist.spotify_artist_id = "sp-artist" + artist.deezer_artist_id = "dz-artist" + + scanner = _build_scanner({"tracks": {"items": []}}, [artist]) + scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False + scanner._get_lookback_period_setting = lambda: "30" + scanner._get_rescan_cutoff = lambda: None + + result = scanner.get_artist_discography_for_watchlist(artist, None) + + assert result is not None + assert result.source == "spotify" + assert result.artist_id == "sp-artist" + assert result.albums and result.albums[0].id == "sp-album" + assert deezer_client.album_calls + assert spotify_client.album_calls + + +def test_match_to_spotify_uses_strict_lookup(): + spotify_client = _FakeSpotifyClient( + search_results=[types.SimpleNamespace(id="fallback-id", name="Artist One")] + ) + scanner = WatchlistScanner(metadata_service=_FakeMetadataService(None, spotify_client=spotify_client)) + original_get_client_for_source = watchlist_scanner_module.get_client_for_source + watchlist_scanner_module.get_client_for_source = lambda source: spotify_client if source == "spotify" else None + + try: + result = scanner._match_to_spotify("Artist One") + finally: + watchlist_scanner_module.get_client_for_source = original_get_client_for_source + + assert result is None + assert spotify_client.search_calls == [("Artist One", 5, False)] diff --git a/web_server.py b/web_server.py index 580ba864..952d9108 100644 --- a/web_server.py +++ b/web_server.py @@ -39906,9 +39906,6 @@ def start_watchlist_scan(): database = get_database() watchlist_artists = database.get_watchlist_artists(profile_id=scan_profile_id) - # Apply global overrides if enabled - _apply_watchlist_global_overrides(watchlist_artists) - if not watchlist_artists: watchlist_scan_state['status'] = 'completed' watchlist_scan_state['summary'] = { @@ -39943,82 +39940,10 @@ def start_watchlist_scan(): except Exception as backfill_error: print(f"Error during {_bf_provider} ID backfilling: {backfill_error}") # Continue with next provider - - # IMAGE BACKFILL — fix watchlist artists with missing images - # Uses DB-only lookups (metadata cache + album art) — no API calls try: - conn = database._get_connection() - cursor = conn.cursor() - cursor.execute(""" - SELECT id, artist_name, spotify_artist_id, itunes_artist_id, - deezer_artist_id, discogs_artist_id - FROM watchlist_artists - WHERE profile_id = ? AND (image_url IS NULL OR image_url = '' OR image_url = 'None' - OR image_url NOT LIKE 'http%') - """, (scan_profile_id,)) - imageless = cursor.fetchall() - - if imageless: - print(f"Backfilling images for {len(imageless)} watchlist artists...") - filled = 0 - for row in imageless: - name = row['artist_name'] - nn = name.lower().strip() - img = None - - # 1. Check metadata cache for artist image - cursor.execute(""" - SELECT image_url FROM metadata_cache_entities - WHERE entity_type = 'artist' AND name = ? COLLATE NOCASE - AND image_url IS NOT NULL AND image_url LIKE 'http%' - LIMIT 1 - """, (name,)) - cr = cursor.fetchone() - if cr: - img = cr['image_url'] - - # 2. Deezer direct URL (no API call needed) - if not img and row['deezer_artist_id']: - img = f"https://api.deezer.com/artist/{row['deezer_artist_id']}/image?size=big" - - # 3. Deezer ID from cache (artist may have a Deezer match we haven't stored on watchlist) - if not img: - cursor.execute(""" - SELECT entity_id FROM metadata_cache_entities - WHERE entity_type = 'artist' AND source = 'deezer' - AND name = ? COLLATE NOCASE LIMIT 1 - """, (name,)) - dz = cursor.fetchone() - if dz and dz['entity_id']: - img = f"https://api.deezer.com/artist/{dz['entity_id']}/image?size=big" - - # 4. Album art fallback (iTunes artists have no artist images) - if not img: - cursor.execute(""" - SELECT image_url FROM metadata_cache_entities - WHERE entity_type = 'album' AND image_url LIKE 'http%' - AND artist_name = ? COLLATE NOCASE LIMIT 1 - """, (name,)) - alb = cursor.fetchone() - if alb: - img = alb['image_url'] - - if img: - aid = (row['spotify_artist_id'] or row['itunes_artist_id'] - or row['deezer_artist_id'] or row['discogs_artist_id']) - if aid: - database.update_watchlist_artist_image(aid, img) - else: - # No external IDs — update by internal row ID directly - cursor.execute(""" - UPDATE watchlist_artists SET image_url = ?, updated_at = CURRENT_TIMESTAMP - WHERE id = ? - """, (img, row['id'])) - conn.commit() - filled += 1 - - if filled: - print(f"Backfilled {filled}/{len(imageless)} watchlist artist images") + filled = scanner.backfill_watchlist_artist_images(scan_profile_id) + if filled: + print(f"Backfilled {filled} watchlist artist images") except Exception as img_err: print(f"Image backfill error: {img_err}") @@ -40043,247 +39968,26 @@ def start_watchlist_scan(): # Pause enrichment workers during scan to reduce API contention _ew_state = _pause_enrichment_workers('watchlist scan') - - # Dynamic delay calculation based on scan scope - lookback_period = scanner._get_lookback_period_setting() - is_full_discography = (lookback_period == 'all') - artist_count = len(watchlist_artists) - - base_artist_delay = 2.0 - base_album_delay = 0.5 - - # Scale up for full discography (way more albums per artist) - if is_full_discography: - base_artist_delay *= 2.0 - base_album_delay *= 2.0 - - # Scale up further for large artist counts (sustained API pressure) - if artist_count > 200: - base_artist_delay *= 1.5 - base_album_delay *= 1.25 - elif artist_count > 100: - base_artist_delay *= 1.25 - - artist_delay = base_artist_delay - album_delay = base_album_delay - print(f"Scan parameters: {artist_count} artists, lookback={lookback_period}, " - f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album") - - # Circuit breaker: pause scan on consecutive rate-limit failures - consecutive_failures = 0 - CIRCUIT_BREAKER_THRESHOLD = 3 - circuit_breaker_pause = 60 # seconds, doubles each trigger, max 600s - - for i, artist in enumerate(watchlist_artists): - # Check for cancel request - if watchlist_scan_state.get('cancel_requested'): - print(f"[Manual Watchlist Scan] Cancel requested after {i}/{len(watchlist_artists)} artists") - watchlist_scan_state['status'] = 'cancelled' - watchlist_scan_state['current_phase'] = 'cancelled' - watchlist_scan_state['summary'] = { - 'total_artists': i, - 'successful_scans': len([r for r in scan_results if r.success]), - 'new_tracks_found': sum(r.new_tracks_found for r in scan_results if r.success), - 'tracks_added_to_wishlist': sum(r.tracks_added_to_wishlist for r in scan_results if r.success), - 'cancelled': True - } - break - - try: - # Fetch artist image using provider-aware method - artist_image_url = scanner.get_artist_image_url(artist) or '' - - # Update progress - watchlist_scan_state.update({ - 'current_artist_index': i + 1, - 'current_artist_name': artist.artist_name, - 'current_artist_image_url': artist_image_url, - 'current_phase': 'fetching_discography', - 'albums_to_check': 0, - 'albums_checked': 0, - 'current_album': '', - 'current_album_image_url': '', - 'current_track_name': '' - }) - - # Get artist discography using provider-aware method - albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp) - - if albums is None: - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': 0, - 'new_tracks_found': 0, - 'tracks_added_to_wishlist': 0, - 'success': False, - 'error_message': "Failed to get artist discography" - })()) - continue - - # Update with album count - watchlist_scan_state.update({ - 'current_phase': 'checking_albums', - 'albums_to_check': len(albums), - 'albums_checked': 0 - }) - - # Track progress for this artist - artist_new_tracks = 0 - artist_added_tracks = 0 - - # Scan each album - for album_index, album in enumerate(albums): - try: - # Get album tracks using provider-aware method - album_data = scanner.metadata_service.get_album(album.id) - if not album_data or 'tracks' not in album_data: - logger.debug(f"Skipping album {album.name} (id={album.id}): no track data returned") - continue - - tracks = album_data['tracks']['items'] - - # Check release type filter (album/EP/single) - if not scanner._should_include_release(len(tracks), artist): - continue - - # Get album image - album_image_url = '' - if 'images' in album_data and album_data['images']: - album_image_url = album_data['images'][0]['url'] - - watchlist_scan_state.update({ - 'albums_checked': album_index + 1, - 'current_album': album.name, - 'current_album_image_url': album_image_url, - 'current_phase': f'checking_album_{album_index + 1}_of_{len(albums)}' - }) - - # Check each track - for track in tracks: - # Check content type filter (live/remix/acoustic/compilation) - if not scanner._should_include_track(track, album_data, artist): - continue - - # Update current track being processed - track_name = track.get('name', 'Unknown Track') - watchlist_scan_state['current_track_name'] = track_name - - if scanner.is_track_missing_from_library(track): - artist_new_tracks += 1 - watchlist_scan_state['tracks_found_this_scan'] += 1 - - # Add to wishlist - if scanner.add_track_to_wishlist(track, album_data, artist): - artist_added_tracks += 1 - watchlist_scan_state['tracks_added_this_scan'] += 1 - - # Add to recent wishlist additions feed - track_artists = track.get('artists', []) - track_artist_name = track_artists[0].get('name', 'Unknown Artist') if track_artists else 'Unknown Artist' - - watchlist_scan_state['recent_wishlist_additions'].insert(0, { - 'track_name': track_name, - 'artist_name': track_artist_name, - 'album_image_url': album_image_url - }) - - # Keep only last 10 - if len(watchlist_scan_state['recent_wishlist_additions']) > 10: - watchlist_scan_state['recent_wishlist_additions'].pop() - - # Rate-limited delay between albums - import time - time.sleep(album_delay) - - except Exception as e: - print(f"Error checking album {album.name}: {e}") - continue - - # Update scan timestamp - scanner.update_artist_scan_timestamp(artist) - - # Store result - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': len(albums), - 'new_tracks_found': artist_new_tracks, - 'tracks_added_to_wishlist': artist_added_tracks, - 'success': True, - 'error_message': None - })()) - - print(f"Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist") - - # Fetch similar artists for discovery feature - # This is critical for the discover page to work - try: - watchlist_scan_state['current_phase'] = 'fetching_similar_artists' - source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id) - - # If Spotify is authenticated, also require Spotify IDs to be present - spotify_authenticated = spotify_client and spotify_client.is_spotify_authenticated() - if database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=scan_profile_id): - print(f" Similar artists for {artist.artist_name} are cached and fresh") - # Still backfill missing iTunes IDs - scanner._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=scan_profile_id) - else: - print(f" Fetching similar artists for {artist.artist_name}...") - scanner.update_similar_artists(artist, profile_id=scan_profile_id) - print(f" Similar artists updated for {artist.artist_name}") - except Exception as similar_error: - print(f" Failed to update similar artists for {artist.artist_name}: {similar_error}") - - # Delay between artists - if i < len(watchlist_artists) - 1: - watchlist_scan_state['current_phase'] = 'rate_limiting' - time.sleep(artist_delay) - - # Reset circuit breaker on successful artist scan - consecutive_failures = 0 - circuit_breaker_pause = 60 - - except Exception as e: - print(f"Error scanning artist {artist.artist_name}: {e}") - - # Circuit breaker: detect consecutive rate-limit failures - error_str = str(e).lower() - if "429" in error_str or "rate limit" in error_str: - consecutive_failures += 1 - if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD: - print(f"Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s") - watchlist_scan_state['current_phase'] = 'circuit_breaker_pause' - time.sleep(circuit_breaker_pause) - circuit_breaker_pause = min(circuit_breaker_pause * 2, 600) - consecutive_failures = 0 - else: - consecutive_failures = 0 - - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': 0, - 'new_tracks_found': 0, - 'tracks_added_to_wishlist': 0, - 'success': False, - 'error_message': str(e) - })()) + scan_results = scanner.scan_watchlist_profile( + scan_profile_id, + watchlist_artists=watchlist_artists, + scan_state=watchlist_scan_state, + cancel_check=lambda: watchlist_scan_state.get('cancel_requested', False), + ) # Store final results (skip if cancelled — already set by cancel handler) was_cancelled = watchlist_scan_state.get('cancel_requested', False) if not was_cancelled: - watchlist_scan_state['status'] = 'completed' - watchlist_scan_state['results'] = scan_results - watchlist_scan_state['completed_at'] = datetime.now() _artmap_cache_invalidate(scan_profile_id) - watchlist_scan_state['current_phase'] = 'completed' - - # Calculate summary successful_scans = [r for r in scan_results if r.success] total_new_tracks = sum(r.new_tracks_found for r in successful_scans) total_added_to_wishlist = sum(r.tracks_added_to_wishlist for r in successful_scans) + watchlist_scan_state['status'] = 'completed' + watchlist_scan_state['results'] = scan_results + watchlist_scan_state['completed_at'] = datetime.now() + watchlist_scan_state['current_phase'] = 'completed' + watchlist_scan_state['summary'] = { 'total_artists': len(scan_results), 'successful_scans': len(successful_scans), @@ -40361,6 +40065,15 @@ def start_watchlist_scan(): import traceback traceback.print_exc() + # Generate Last.fm radio playlists (weekly refresh) + print("Starting Last.fm radio generation...") + watchlist_scan_state['current_phase'] = 'generating_lastfm_radio' + try: + scanner._generate_lastfm_radio_playlists() + print("Last.fm radio generation complete") + except Exception as lastfm_error: + print(f"Error generating Last.fm radio playlists: {lastfm_error}") + # Sync Spotify library cache print("Syncing Spotify library cache...") watchlist_scan_state['current_phase'] = 'syncing_spotify_library' @@ -40877,33 +40590,6 @@ def watchlist_global_config(): traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 -def _apply_watchlist_global_overrides(watchlist_artists): - """If global override is enabled, overwrite per-artist settings on WatchlistArtist objects.""" - if not config_manager.get('watchlist.global_override_enabled', False): - return - # Read global settings once - g_albums = config_manager.get('watchlist.global_include_albums', True) - g_eps = config_manager.get('watchlist.global_include_eps', True) - g_singles = config_manager.get('watchlist.global_include_singles', True) - g_live = config_manager.get('watchlist.global_include_live', False) - g_remixes = config_manager.get('watchlist.global_include_remixes', False) - g_acoustic = config_manager.get('watchlist.global_include_acoustic', False) - g_compilations = config_manager.get('watchlist.global_include_compilations', False) - g_instrumentals = config_manager.get('watchlist.global_include_instrumentals', False) - print(f"[Watchlist] Global override is ACTIVE — applying to {len(watchlist_artists)} artists " - f"(albums={g_albums}, eps={g_eps}, singles={g_singles}, live={g_live}, " - f"remixes={g_remixes}, acoustic={g_acoustic}, compilations={g_compilations}, " - f"instrumentals={g_instrumentals})") - for artist in watchlist_artists: - artist.include_albums = g_albums - artist.include_eps = g_eps - artist.include_singles = g_singles - artist.include_live = g_live - artist.include_remixes = g_remixes - artist.include_acoustic = g_acoustic - artist.include_compilations = g_compilations - artist.include_instrumentals = g_instrumentals - def _update_similar_artists_worker(): """Background worker to update similar artists for all watchlist artists""" global similar_artists_update_state @@ -41058,8 +40744,13 @@ def _process_watchlist_scan_automatically(automation_id=None, profile_id=None): scanner = get_watchlist_scanner(spotify_client) all_profiles = scan_profiles # Used later for discovery pool population - # Apply global overrides if enabled - _apply_watchlist_global_overrides(watchlist_artists) + for p in scan_profiles: + try: + filled = scanner.backfill_watchlist_artist_images(p['id']) + if filled: + print(f"Backfilled {filled} watchlist artist images for profile {p['id']}") + except Exception as img_err: + print(f"Image backfill error for profile {p['id']}: {img_err}") # Initialize detailed progress tracking (same as manual scan) watchlist_scan_state = { @@ -41089,259 +40780,79 @@ def _process_watchlist_scan_automatically(automation_id=None, profile_id=None): # Pause enrichment workers during scan to reduce API contention _ew_state = _pause_enrichment_workers('auto-watchlist scan') - # Dynamic delay calculation based on scan scope - lookback_period = scanner._get_lookback_period_setting() - is_full_discography = (lookback_period == 'all') - artist_count = len(watchlist_artists) - - base_artist_delay = 2.0 - base_album_delay = 0.5 - - # Scale up for full discography (way more albums per artist) - if is_full_discography: - base_artist_delay *= 2.0 - base_album_delay *= 2.0 - - # Scale up further for large artist counts (sustained API pressure) - if artist_count > 200: - base_artist_delay *= 1.5 - base_album_delay *= 1.25 - elif artist_count > 100: - base_artist_delay *= 1.25 - - artist_delay = base_artist_delay - album_delay = base_album_delay - print(f"[Auto-Watchlist] Scan parameters: {artist_count} artists, lookback={lookback_period}, " - f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album") - - # Circuit breaker: pause scan on consecutive rate-limit failures - consecutive_failures = 0 - CIRCUIT_BREAKER_THRESHOLD = 3 - circuit_breaker_pause = 60 # seconds, doubles each trigger, max 600s - - # Scan each artist with detailed tracking - for i, artist in enumerate(watchlist_artists): - # Check for cancel request - if watchlist_scan_state.get('cancel_requested'): - print(f"[Auto-Watchlist] Cancel requested after {i}/{len(watchlist_artists)} artists") - watchlist_scan_state['status'] = 'cancelled' - watchlist_scan_state['current_phase'] = 'cancelled' - watchlist_scan_state['summary'] = { - 'total_artists': i, - 'successful_scans': len([r for r in scan_results if r.success]), - 'new_tracks_found': sum(r.new_tracks_found for r in scan_results if r.success), - 'tracks_added_to_wishlist': sum(r.tracks_added_to_wishlist for r in scan_results if r.success), - 'cancelled': True - } - _update_automation_progress(automation_id, progress=100, phase='Cancelled by user', - log_line='Scan cancelled by user', log_type='warning') - break - - try: - # Fetch artist image using provider-aware method - artist_image_url = scanner.get_artist_image_url(artist) or '' - - pct = 5 + (i / max(1, len(watchlist_artists))) * 90 - _update_automation_progress(automation_id, progress=pct, - phase=f'Scanning: {artist.artist_name} ({i+1}/{len(watchlist_artists)})', - current_item=artist.artist_name, - processed=i, total=len(watchlist_artists)) - - # Update progress - watchlist_scan_state.update({ - 'current_artist_index': i + 1, - 'current_artist_name': artist.artist_name, - 'current_artist_image_url': artist_image_url, - 'current_phase': 'fetching_discography', - 'albums_to_check': 0, - 'albums_checked': 0, - 'current_album': '', - 'current_album_image_url': '', - 'current_track_name': '' - }) - - # Get artist discography using provider-aware method - albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp) - - if albums is None: - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': 0, - 'new_tracks_found': 0, - 'tracks_added_to_wishlist': 0, - 'success': False, - 'error_message': "Failed to get artist discography" - })()) - continue - - # Update with album count - watchlist_scan_state.update({ - 'current_phase': 'checking_albums', - 'albums_to_check': len(albums), - 'albums_checked': 0 - }) - - # Track progress for this artist - artist_new_tracks = 0 - artist_added_tracks = 0 - - # Scan each album - for album_index, album in enumerate(albums): - try: - # Get album tracks using provider-aware method - album_data = scanner.metadata_service.get_album(album.id) - if not album_data or 'tracks' not in album_data: - logger.debug(f"Skipping album {album.name} (id={album.id}): no track data returned") - continue - - tracks = album_data['tracks']['items'] - - # Check release type filter (album/EP/single) - if not scanner._should_include_release(len(tracks), artist): - continue - - # Get album image - album_image_url = '' - if 'images' in album_data and album_data['images']: - album_image_url = album_data['images'][0]['url'] - - watchlist_scan_state.update({ - 'albums_checked': album_index + 1, - 'current_album': album.name, - 'current_album_image_url': album_image_url, - 'current_phase': f'checking_album_{album_index + 1}_of_{len(albums)}' - }) - - # Check each track - for track in tracks: - # Check content type filter (live/remix/acoustic/compilation) - if not scanner._should_include_track(track, album_data, artist): - continue - - # Update current track being processed - track_name = track.get('name', 'Unknown Track') - watchlist_scan_state['current_track_name'] = track_name - - if scanner.is_track_missing_from_library(track): - artist_new_tracks += 1 - watchlist_scan_state['tracks_found_this_scan'] += 1 - - # Add to wishlist - if scanner.add_track_to_wishlist(track, album_data, artist): - artist_added_tracks += 1 - watchlist_scan_state['tracks_added_this_scan'] += 1 - - # Add to recent wishlist additions feed - track_artists = track.get('artists', []) - track_artist_name = track_artists[0].get('name', 'Unknown Artist') if track_artists else 'Unknown Artist' - - watchlist_scan_state['recent_wishlist_additions'].insert(0, { - 'track_name': track_name, - 'artist_name': track_artist_name, - 'album_image_url': album_image_url - }) - - # Keep only last 10 - if len(watchlist_scan_state['recent_wishlist_additions']) > 10: - watchlist_scan_state['recent_wishlist_additions'].pop() - - # Rate-limited delay between albums - import time - time.sleep(album_delay) - - except Exception as e: - print(f"Error checking album {album.name}: {e}") - continue - - # Update scan timestamp - scanner.update_artist_scan_timestamp(artist) - - # Store result - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': len(albums), - 'new_tracks_found': artist_new_tracks, - 'tracks_added_to_wishlist': artist_added_tracks, - 'success': True, - 'error_message': None - })()) - - print(f"Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist") - if artist_new_tracks > 0: - _update_automation_progress(automation_id, - log_line=f'{artist.artist_name} — {artist_new_tracks} new, {artist_added_tracks} added', log_type='success') + def _scan_progress(event_type, payload): + if event_type == 'scan_started': + _update_automation_progress( + automation_id, + progress=5, + phase='Loading watchlist', + log_line=f"{len(watchlist_artists)} artists ({profile_label})", + log_type='info', + ) + elif event_type == 'artist_started': + total = max(1, payload.get('total_artists', len(watchlist_artists))) + idx = payload.get('artist_index', 1) + artist_name = payload.get('artist_name', '') + pct = 5 + ((idx - 1) / total) * 90 + _update_automation_progress( + automation_id, + progress=pct, + phase=f'Scanning: {artist_name} ({idx}/{total})', + current_item=artist_name, + processed=idx - 1, + total=total, + ) + elif event_type == 'artist_completed': + artist_name = payload.get('artist_name', '') + new_tracks = payload.get('new_tracks_found', 0) + added = payload.get('tracks_added_to_wishlist', 0) + if new_tracks > 0: + _update_automation_progress( + automation_id, + log_line=f'{artist_name} — {new_tracks} new, {added} added', + log_type='success', + ) else: - _update_automation_progress(automation_id, - log_line=f'{artist.artist_name} — no new tracks', log_type='skip') + _update_automation_progress( + automation_id, + log_line=f'{artist_name} — no new tracks', + log_type='skip', + ) + elif event_type == 'artist_error': + artist_name = payload.get('artist_name', '') + error_message = payload.get('error_message', 'error') + _update_automation_progress( + automation_id, + log_line=f'{artist_name} — error: {error_message[:60]}', + log_type='error', + ) + elif event_type == 'cancelled': + _update_automation_progress( + automation_id, + progress=100, + phase='Cancelled by user', + log_line='Scan cancelled by user', + log_type='warning', + ) + elif event_type == 'scan_completed': + _update_automation_progress( + automation_id, + progress=95, + phase='Scan complete', + log_line=( + f"Scanned {payload.get('successful_scans', 0)} artists — " + f"{payload.get('new_tracks_found', 0)} new tracks, " + f"{payload.get('tracks_added_to_wishlist', 0)} added to wishlist" + ), + log_type='success' if payload.get('new_tracks_found', 0) > 0 else 'info', + ) - # Emit watchlist_new_release event if new tracks were found - if artist_new_tracks > 0: - try: - if automation_engine: - automation_engine.emit('watchlist_new_release', { - 'artist': artist.artist_name, - 'new_tracks': str(artist_new_tracks), - 'added_to_wishlist': str(artist_added_tracks), - }) - except Exception: - pass - - # Fetch similar artists for discovery feature (per-profile) - try: - watchlist_scan_state['current_phase'] = 'fetching_similar_artists' - source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id) - artist_profile_id = getattr(artist, 'profile_id', 1) - - spotify_authenticated = spotify_client and spotify_client.is_spotify_authenticated() - if database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=artist_profile_id): - print(f" Similar artists for {artist.artist_name} are cached and fresh (profile {artist_profile_id})") - scanner._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=artist_profile_id) - else: - print(f" Fetching similar artists for {artist.artist_name} (profile {artist_profile_id})...") - scanner.update_similar_artists(artist, profile_id=artist_profile_id) - print(f" Similar artists updated for {artist.artist_name}") - except Exception as similar_error: - print(f" Failed to update similar artists for {artist.artist_name}: {similar_error}") - - # Delay between artists - if i < len(watchlist_artists) - 1: - watchlist_scan_state['current_phase'] = 'rate_limiting' - time.sleep(artist_delay) - - # Reset circuit breaker on successful artist scan - consecutive_failures = 0 - circuit_breaker_pause = 60 - - except Exception as e: - print(f"Error scanning artist {artist.artist_name}: {e}") - _update_automation_progress(automation_id, - log_line=f'{artist.artist_name} — error: {str(e)[:60]}', log_type='error') - - # Circuit breaker: detect consecutive rate-limit failures - error_str = str(e).lower() - if "429" in error_str or "rate limit" in error_str: - consecutive_failures += 1 - if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD: - print(f"[Auto-Watchlist] Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s") - watchlist_scan_state['current_phase'] = 'circuit_breaker_pause' - time.sleep(circuit_breaker_pause) - circuit_breaker_pause = min(circuit_breaker_pause * 2, 600) - consecutive_failures = 0 - else: - consecutive_failures = 0 - - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': 0, - 'new_tracks_found': 0, - 'tracks_added_to_wishlist': 0, - 'success': False, - 'error_message': str(e) - })()) - continue + scan_results = scanner.scan_watchlist_artists( + watchlist_artists, + scan_state=watchlist_scan_state, + progress_callback=_scan_progress, + cancel_check=lambda: watchlist_scan_state.get('cancel_requested'), + ) # Update state with results (skip if cancelled — already set by cancel handler) was_cancelled = watchlist_scan_state.get('cancel_requested', False) @@ -41477,6 +40988,21 @@ def _process_watchlist_scan_automatically(automation_id=None, profile_id=None): _update_automation_progress(automation_id, log_line=f'Seasonal error: {seasonal_error}', log_type='error') + # Generate Last.fm radio playlists (weekly refresh) + print("Starting Last.fm radio generation...") + watchlist_scan_state['current_phase'] = 'generating_lastfm_radio' + _update_automation_progress(automation_id, progress=99, phase='Generating Last.fm radio', + log_line='Building Last.fm radio playlists...', log_type='info') + try: + scanner._generate_lastfm_radio_playlists() + print("Last.fm radio generation complete") + _update_automation_progress(automation_id, + log_line='Last.fm radio playlists updated', log_type='success') + except Exception as lastfm_error: + print(f"Error generating Last.fm radio playlists: {lastfm_error}") + _update_automation_progress(automation_id, + log_line=f'Last.fm radio error: {lastfm_error}', log_type='error') + # Sync Spotify library cache print("Syncing Spotify library cache...") try: @@ -42001,7 +41527,7 @@ def refresh_spotify_library(): def _run_sync(): try: from core.watchlist_scanner import get_watchlist_scanner - scanner = get_watchlist_scanner() + scanner = get_watchlist_scanner(spotify_client) if scanner: # Force full sync by clearing last_sync timestamp database = get_database()