diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 1d3f461f..2cdc9a03 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -30,63 +30,6 @@ DELAY_BETWEEN_ARTISTS = 4.0 # 4 seconds between different artists (was 2s, DELAY_BETWEEN_ALBUMS = 0.5 # 500ms between albums for same artist DELAY_BETWEEN_API_BATCHES = 1.0 # 1 second between API batch operations -# iTunes API retry configuration -ITUNES_MAX_RETRIES = 3 -ITUNES_BASE_DELAY = 1.0 # Base delay in seconds for exponential backoff - - -def _get_fallback_metadata_client(): - """Get the configured metadata client — delegates to centralized metadata_service.""" - from core.metadata_service import get_primary_source, get_primary_client - return get_primary_client(), get_primary_source() - - -def itunes_api_call_with_retry(func, *args, max_retries=ITUNES_MAX_RETRIES, **kwargs): - """ - Execute an iTunes API call with exponential backoff retry logic. - - Args: - func: The function to call - *args: Arguments to pass to the function - max_retries: Maximum number of retry attempts - **kwargs: Keyword arguments to pass to the function - - Returns: - The result of the function call, or None if all retries failed - """ - last_error = None - for attempt in range(max_retries): - try: - result = func(*args, **kwargs) - return result - except requests.exceptions.HTTPError as e: - # Handle rate limiting (429) and server errors (5xx) - if e.response is not None and e.response.status_code == 429: - delay = ITUNES_BASE_DELAY * (2 ** attempt) - logger.warning(f"[iTunes] Rate limited, retrying in {delay}s (attempt {attempt + 1}/{max_retries})") - time.sleep(delay) - last_error = e - elif e.response is not None and e.response.status_code >= 500: - delay = ITUNES_BASE_DELAY * (2 ** attempt) - logger.warning(f"[iTunes] Server error {e.response.status_code}, retrying in {delay}s (attempt {attempt + 1}/{max_retries})") - time.sleep(delay) - last_error = e - else: - raise # Don't retry on client errors (4xx except 429) - except requests.exceptions.RequestException as e: - # Retry on connection errors - delay = ITUNES_BASE_DELAY * (2 ** attempt) - logger.warning(f"[iTunes] Connection error, retrying in {delay}s (attempt {attempt + 1}/{max_retries}): {e}") - time.sleep(delay) - last_error = e - except Exception as e: - # Don't retry on other exceptions - raise - - if last_error: - logger.error(f"[iTunes] All {max_retries} retry attempts failed: {last_error}") - return None - def clean_track_name_for_search(track_name): """ @@ -514,17 +457,7 @@ class WatchlistScanner: if stored_id: return stored_id - if not client or not hasattr(client, 'search_artists'): - return None - - try: - search_kwargs = {'limit': 1} - if source == 'spotify': - search_kwargs['allow_fallback'] = False - search_results = client.search_artists(watchlist_artist.artist_name, **search_kwargs) - except Exception as e: - logger.debug("Could not search %s for %s: %s", source, watchlist_artist.artist_name, e) - return None + search_results = self._search_artists_for_source(source, watchlist_artist.artist_name, limit=1, client=client) if not search_results: return None @@ -534,6 +467,22 @@ class WatchlistScanner: self._cache_watchlist_artist_source_id(watchlist_artist, source, found_id) return found_id + def _search_artists_for_source(self, source: str, artist_name: str, limit: int = 1, client: Any = None) -> List[Any]: + """Search artists for a specific source, keeping Spotify strict.""" + if client is None: + client = get_client_for_source(source) + if not client or not hasattr(client, 'search_artists'): + return [] + + try: + search_kwargs = {'limit': limit} + if source == 'spotify': + search_kwargs['allow_fallback'] = False + return client.search_artists(artist_name, **search_kwargs) or [] + except Exception as e: + logger.debug("Could not search %s for %s: %s", source, artist_name, e) + return [] + @staticmethod def _get_artist_image_from_data(artist_data: Any) -> Optional[str]: """Extract an image URL from artist payloads across providers.""" @@ -566,6 +515,41 @@ class WatchlistScanner: or getattr(artist_data, 'cover_image', None) ) + def _get_artist_metadata_from_data(self, artist_data: Any) -> Dict[str, Any]: + """Extract normalized artist metadata from a provider result.""" + if not artist_data: + return {'name': None, 'image_url': None, 'genres': [], 'popularity': 0} + + if isinstance(artist_data, dict): + name = artist_data.get('name') or artist_data.get('artist_name') or artist_data.get('title') + genres = artist_data.get('genres') or [] + popularity = artist_data.get('popularity') or artist_data.get('rank') or 0 + else: + name = ( + getattr(artist_data, 'name', None) + or getattr(artist_data, 'artist_name', None) + or getattr(artist_data, 'title', None) + ) + genres = getattr(artist_data, 'genres', None) or [] + popularity = getattr(artist_data, 'popularity', None) or getattr(artist_data, 'rank', None) or 0 + + if isinstance(genres, str): + genres = [genres] + elif not isinstance(genres, list): + genres = list(genres) if genres else [] + + try: + popularity = int(popularity or 0) + except Exception: + popularity = 0 + + return { + 'name': name, + 'image_url': self._get_artist_image_from_data(artist_data), + 'genres': genres, + 'popularity': popularity, + } + def _get_artist_image_for_source(self, watchlist_artist: WatchlistArtist, source: str, client: Any, artist_id: str) -> Optional[str]: """Fetch an artist image for a specific source.""" if not client or not artist_id or not hasattr(client, 'get_artist'): @@ -875,7 +859,7 @@ class WatchlistScanner: Returns the first provider that can actually return albums. Args: - watchlist_artist: WatchlistArtist object (has both spotify and itunes IDs) + watchlist_artist: WatchlistArtist object (has provider IDs when available) last_scan_timestamp: Only return releases after this date (for incremental scans) Returns: @@ -1271,10 +1255,9 @@ class WatchlistScanner: if scan_state is not None: scan_state['current_phase'] = 'fetching_similar_artists' artist_profile_id = getattr(artist, 'profile_id', profile_id) - spotify_authenticated = self.spotify_client and self.spotify_client.is_spotify_authenticated() - if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=artist_profile_id): + if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30, profile_id=artist_profile_id): logger.info("Similar artists for %s are cached and fresh (profile %s)", artist.artist_name, artist_profile_id) - self._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=artist_profile_id) + self._backfill_similar_artists_fallback_ids(source_artist_id, profile_id=artist_profile_id) else: logger.info("Fetching similar artists for %s (profile %s)...", artist.artist_name, artist_profile_id) self.update_similar_artists(artist, profile_id=artist_profile_id, source_artist_id=source_artist_id) @@ -2122,20 +2105,22 @@ class WatchlistScanner: def _fetch_similar_artists_from_musicmap(self, artist_name: str, limit: int = 20) -> List[Dict[str, Any]]: """ - Fetch similar artists from MusicMap and match them to both Spotify and iTunes. + Fetch similar artists from MusicMap and match them against configured metadata providers. Args: artist_name: The artist name to find similar artists for limit: Maximum number of similar artists to return (default: 20) Returns: - List of matched artist dictionaries with both Spotify and iTunes IDs when available + List of matched artist dictionaries with provider-specific IDs when available """ try: logger.info(f"Fetching similar artists from MusicMap for: {artist_name}") # Construct MusicMap URL - url_artist = artist_name.lower().replace(' ', '+') + from urllib.parse import quote_plus + + url_artist = quote_plus(artist_name.strip()) musicmap_url = f'https://www.music-map.com/{url_artist}' # Set headers to mimic a browser @@ -2173,36 +2158,33 @@ class WatchlistScanner: logger.info(f"Found {len(similar_artist_names)} similar artists from MusicMap") - # Get fallback metadata client for matching (iTunes or Deezer) - itunes_client, fallback_source = _get_fallback_metadata_client() + source_priority = self._discovery_source_priority() + source_id_keys = { + 'spotify': 'spotify_id', + 'itunes': 'itunes_id', + 'deezer': 'deezer_id', + } + searched_source_ids = {} + available_sources = [] - # Get the searched artist's IDs to exclude them - searched_spotify_id = None - searched_fallback_id = None - try: - # Try Spotify search (only when Spotify is the configured primary source) - if self._spotify_is_primary_source(): - searched_results = self.spotify_client.search_artists(artist_name, limit=1, allow_fallback=False) - if searched_results and len(searched_results) > 0: - searched_spotify_id = searched_results[0].id - except Exception as e: - logger.debug(f"Could not get searched artist Spotify ID: {e}") + for source in source_priority: + search_results = self._search_artists_for_source(source, artist_name, limit=1) + if search_results: + searched_source_ids[source] = self._extract_entity_id(search_results[0]) + available_sources.append(source) + else: + searched_source_ids[source] = None - try: - # Try fallback source (iTunes/Deezer) search - fallback_results = itunes_client.search_artists(artist_name, limit=1) - if fallback_results and len(fallback_results) > 0: - searched_fallback_id = fallback_results[0].id - except Exception as e: - logger.debug(f"Could not get searched artist {fallback_source} ID: {e}") + if not available_sources: + logger.warning(f"No metadata providers available for MusicMap matching: {artist_name}") + return [] - # Match each artist to both Spotify and fallback source (iTunes/Deezer) matched_artists = [] - seen_names = set() # Track seen artist names to prevent duplicates + seen_names = set() + provider_match_counts = {source: 0 for source in available_sources} for artist_name_to_match in similar_artist_names[:limit]: try: - # Skip if we've already matched this artist name name_lower = artist_name_to_match.lower().strip() if name_lower in seen_names: continue @@ -2214,71 +2196,56 @@ class WatchlistScanner: 'deezer_id': None, 'image_url': None, 'genres': [], - 'popularity': 0 + 'popularity': 0, } - # Try to match on Spotify (only when Spotify is the configured primary source) - if self._spotify_is_primary_source(): - try: - spotify_results = self.spotify_client.search_artists( - artist_name_to_match, - limit=1, - allow_fallback=False, - ) - if spotify_results and len(spotify_results) > 0: - spotify_artist = spotify_results[0] - # Skip if this is the searched artist - if spotify_artist.id != searched_spotify_id: - artist_data['spotify_id'] = spotify_artist.id - artist_data['name'] = spotify_artist.name # Use canonical name - artist_data['image_url'] = spotify_artist.image_url if hasattr(spotify_artist, 'image_url') else None - artist_data['genres'] = spotify_artist.genres if hasattr(spotify_artist, 'genres') else [] - artist_data['popularity'] = spotify_artist.popularity if hasattr(spotify_artist, 'popularity') else 0 - except Exception as e: - logger.debug(f"Spotify match failed for {artist_name_to_match}: {e}") + for source in available_sources: + search_results = self._search_artists_for_source(source, artist_name_to_match, limit=1) + if not search_results: + continue - # Try to match on fallback source (iTunes/Deezer) with retry for rate limiting - try: - fallback_results = itunes_api_call_with_retry( - itunes_client.search_artists, artist_name_to_match, limit=1 - ) - if fallback_results and len(fallback_results) > 0: - fallback_artist = fallback_results[0] - # Skip if this is the searched artist - if fallback_artist.id != searched_fallback_id: - # Store under the appropriate key based on fallback source - if fallback_source == 'deezer': - artist_data['deezer_id'] = fallback_artist.id - else: - artist_data['itunes_id'] = fallback_artist.id - # Use fallback name if we don't have Spotify - if not artist_data['spotify_id']: - artist_data['name'] = fallback_artist.name - # Use fallback genres if we don't have Spotify genres - if not artist_data['genres'] and hasattr(fallback_artist, 'genres'): - artist_data['genres'] = fallback_artist.genres - else: - logger.info(f" [{fallback_source}] No match found for: {artist_name_to_match}") - except Exception as e: - logger.info(f" [{fallback_source}] Match failed for {artist_name_to_match}: {e}") + matched_artist = search_results[0] + matched_id = self._extract_entity_id(matched_artist) + if not matched_id or matched_id == searched_source_ids.get(source): + continue - # Only add if we got at least one ID - fallback_id_key = 'deezer_id' if fallback_source == 'deezer' else 'itunes_id' - if artist_data['spotify_id'] or artist_data.get(fallback_id_key): + id_key = source_id_keys.get(source) + if not id_key: + continue + + artist_data[id_key] = matched_id + provider_match_counts[source] += 1 + + metadata = self._get_artist_metadata_from_data(matched_artist) + if metadata['name'] and artist_data['name'] == artist_name_to_match: + artist_data['name'] = metadata['name'] + if metadata['image_url'] and not artist_data['image_url']: + artist_data['image_url'] = metadata['image_url'] + if metadata['genres'] and not artist_data['genres']: + artist_data['genres'] = metadata['genres'] + if metadata['popularity'] and not artist_data['popularity']: + artist_data['popularity'] = metadata['popularity'] + + if any(artist_data.get(key) for key in source_id_keys.values()): seen_names.add(name_lower) matched_artists.append(artist_data) - logger.debug(f" Matched: {artist_data['name']} (Spotify: {artist_data['spotify_id']}, {fallback_source}: {artist_data.get(fallback_id_key)})") + provider_summary = ", ".join( + f"{source}: {artist_data.get(source_id_keys[source])}" + for source in available_sources + if artist_data.get(source_id_keys[source]) + ) + logger.debug(f" Matched: {artist_data['name']} ({provider_summary})") except Exception as match_error: logger.debug(f"Error matching {artist_name_to_match}: {match_error}") continue # Log detailed matching statistics - fallback_id_key = 'deezer_id' if fallback_source == 'deezer' else 'itunes_id' - fallback_matched = sum(1 for a in matched_artists if a.get(fallback_id_key)) - spotify_matched = sum(1 for a in matched_artists if a.get('spotify_id')) - both_matched = sum(1 for a in matched_artists if a.get(fallback_id_key) and a.get('spotify_id')) - logger.info(f"Matched {len(matched_artists)} similar artists - {fallback_source}: {fallback_matched}, Spotify: {spotify_matched}, Both: {both_matched}") + provider_stats = ", ".join( + f"{source}: {provider_match_counts[source]}" + for source in available_sources + ) + logger.info(f"Matched {len(matched_artists)} similar artists - {provider_stats}") return matched_artists except requests.exceptions.RequestException as e: @@ -2288,55 +2255,73 @@ class WatchlistScanner: logger.error(f"Error fetching similar artists from MusicMap: {e}") return [] - def _backfill_similar_artists_itunes_ids(self, source_artist_id: str, profile_id: int = 1) -> int: - """ - Backfill missing iTunes IDs for cached similar artists. - This ensures seamless dual-source support without clearing cached data. + def _update_similar_artist_source_id(self, similar_artist_id: int, source: str, source_id: str) -> bool: + """Persist a resolved similar-artist ID for a supported source.""" + if source == 'deezer': + return self.database.update_similar_artist_deezer_id(similar_artist_id, source_id) + if source == 'itunes': + return self.database.update_similar_artist_itunes_id(similar_artist_id, source_id) + return False - Args: - source_artist_id: The source artist ID to backfill similar artists for - profile_id: Profile to scope the backfill to - - Returns: - Number of similar artists updated with iTunes IDs + def _backfill_similar_artists_fallback_ids(self, source_artist_id: str, profile_id: int = 1) -> int: """ + Backfill missing fallback-provider IDs for cached similar artists. + + Uses the configured source priority, filtered to providers that have + writable similar-artist ID columns. This keeps old cached rows usable + when the active metadata provider changes. + """ + backfill_sources = [source for source in self._discovery_source_priority() if source in {'itunes', 'deezer'}] + if not backfill_sources: + logger.debug("No fallback metadata providers available for similar-artist backfill") + return 0 + + updated_total = 0 + try: - # Get fallback metadata client (iTunes or Deezer) - fallback_client, fallback_source = _get_fallback_metadata_client() - - # Get similar artists that are missing IDs for the active fallback source - similar_artists = self.database.get_similar_artists_missing_fallback_ids(source_artist_id, fallback_source, profile_id=profile_id) - - if not similar_artists: - return 0 - - logger.info(f"Backfilling {fallback_source} IDs for {len(similar_artists)} similar artists") - - updated_count = 0 - for similar_artist in similar_artists: - try: - results = fallback_client.search_artists(similar_artist.similar_artist_name, limit=1) - if results and len(results) > 0: - found_id = results[0].id - # Update the similar artist with the correct source ID - if fallback_source == 'deezer': - success = self.database.update_similar_artist_deezer_id(similar_artist.id, found_id) - else: - success = self.database.update_similar_artist_itunes_id(similar_artist.id, found_id) - if success: - updated_count += 1 - logger.debug(f" Backfilled {fallback_source} ID {found_id} for {similar_artist.similar_artist_name}") - except Exception as e: - logger.debug(f" Could not backfill {fallback_source} ID for {similar_artist.similar_artist_name}: {e}") + for source in backfill_sources: + client = get_client_for_source(source) + if not client: + logger.debug("Skipping %s similar-artist backfill - client unavailable", source) continue - if updated_count > 0: - logger.info(f"Backfilled {updated_count} similar artists with {fallback_source} IDs") + similar_artists = self.database.get_similar_artists_missing_fallback_ids( + source_artist_id, + source, + profile_id=profile_id, + ) + if not similar_artists: + continue - return updated_count + logger.info("Backfilling %s IDs for %s similar artists", source, len(similar_artists)) + + updated_count = 0 + for similar_artist in similar_artists: + try: + results = self._search_artists_for_source(source, similar_artist.similar_artist_name, limit=1, client=client) + if not results: + continue + + found_id = self._extract_entity_id(results[0]) + if not found_id: + continue + + success = self._update_similar_artist_source_id(similar_artist.id, source, found_id) + if success: + updated_count += 1 + updated_total += 1 + logger.debug(" Backfilled %s ID %s for %s", source, found_id, similar_artist.similar_artist_name) + except Exception as e: + logger.debug(" Could not backfill %s ID for %s: %s", source, similar_artist.similar_artist_name, e) + continue + + if updated_count > 0: + logger.info("Backfilled %s similar artists with %s IDs", updated_count, source) + + return updated_total except Exception as e: - logger.error(f"Error backfilling similar artists {fallback_source if 'fallback_source' in dir() else 'fallback'} IDs: {e}") + logger.error("Error backfilling similar artists IDs: %s", e) return 0 def update_similar_artists( @@ -2349,12 +2334,12 @@ class WatchlistScanner: """ Fetch and store similar artists for a watchlist artist. Called after each artist scan to build discovery pool. - Uses MusicMap to find similar artists and matches them to both Spotify and iTunes. + Uses MusicMap to find similar artists and matches them against available metadata providers. """ try: logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}") - # Get similar artists from MusicMap (returns list of artist dicts with both IDs) + # Get similar artists from MusicMap (returns list of artist dicts with provider IDs) similar_artists = self._fetch_similar_artists_from_musicmap(watchlist_artist.artist_name, limit=limit) if not similar_artists: @@ -2377,7 +2362,7 @@ class WatchlistScanner: stored_count = 0 for rank, similar_artist in enumerate(similar_artists, 1): try: - # similar_artist has 'name', 'spotify_id', 'itunes_id', 'deezer_id', 'image_url', 'genres', 'popularity' + # similar_artist has 'name', provider IDs, 'image_url', 'genres', 'popularity' success = self.database.add_or_update_similar_artist( source_artist_id=source_artist_id, similar_artist_name=similar_artist['name'], @@ -2413,7 +2398,7 @@ class WatchlistScanner: Populate discovery pool with tracks from top similar artists. Called after watchlist scan completes. - Supports both Spotify and iTunes sources - populates for whichever is available. + Supports Spotify, iTunes, and Deezer sources - populates for whichever is available. - Checks if pool was updated in last 24 hours (prevents over-polling) - Includes albums, singles, and EPs for comprehensive coverage - Appends to existing pool instead of replacing it @@ -2841,6 +2826,11 @@ class WatchlistScanner: logger.info("No watchlist artists to check for incremental update") return + discovery_sources = self._discovery_source_priority() + if not discovery_sources: + logger.warning("No discovery sources available for incremental update") + return + cutoff_date = datetime.now() - timedelta(days=7) # Only last week's releases total_tracks_added = 0 @@ -2848,27 +2838,57 @@ class WatchlistScanner: try: logger.info(f"[{artist_idx}/{len(watchlist_artists)}] Checking {artist.artist_name} for new releases...") - # Only fetch latest 5 releases (much faster than full scan) - recent_releases = self.spotify_client.get_artist_albums( - artist.spotify_artist_id, - album_type='album,single,ep', - limit=5, - skip_cache=True, - max_pages=1, - ) + selected_source = None + selected_artist_id = None + recent_releases = [] + artist_genres: List[str] = [] - if not recent_releases: + for source in discovery_sources: + source_attr = self._artist_id_attribute_for_source(source) + stored_id = getattr(artist, source_attr, None) if source_attr else None + + cache_callback = None + if source == 'spotify': + cache_callback = lambda found_id, watchlist_id=artist.id: self._cache_watchlist_artist_source_id(artist, 'spotify', found_id) + elif source == 'itunes': + cache_callback = lambda found_id, watchlist_id=artist.id: self._cache_watchlist_artist_source_id(artist, 'itunes', found_id) + elif source == 'deezer': + cache_callback = lambda found_id, watchlist_id=artist.id: self._cache_watchlist_artist_source_id(artist, 'deezer', found_id) + + artist_id = self._resolve_artist_id_for_source( + source, + artist.artist_name, + stored_id=stored_id, + cache_callback=cache_callback, + ) + if not artist_id: + continue + + recent_releases = self._get_artist_albums_for_source( + source, + artist_id, + album_type='album,single,ep', + limit=5, + skip_cache=True, + max_pages=1, + ) + if not recent_releases: + continue + + try: + artist_data = self._get_artist_data_for_source(source, artist_id) + if artist_data and 'genres' in artist_data: + artist_genres = artist_data['genres'] + except Exception as e: + logger.debug(f"Could not fetch genres for {artist.artist_name} on {source}: {e}") + + selected_source = source + selected_artist_id = artist_id + break + + if not recent_releases or not selected_source or not selected_artist_id: continue - # Fetch artist genres once for all tracks of this artist - artist_genres = [] - try: - artist_data = self.spotify_client.get_artist(artist.spotify_artist_id) - if artist_data and 'genres' in artist_data: - artist_genres = artist_data['genres'] - except Exception as e: - logger.debug(f"Could not fetch genres for {artist.artist_name}: {e}") - for release in recent_releases: try: # Check if release is within cutoff @@ -2876,7 +2896,7 @@ class WatchlistScanner: continue # Skip older releases # Get full album data with tracks - album_data = self.spotify_client.get_album(release.id) + album_data = self._get_album_data_for_source(selected_source, release.id, album_name=release.name) if not album_data or 'tracks' not in album_data: continue @@ -2926,7 +2946,20 @@ class WatchlistScanner: 'artist_genres': artist_genres } - if self.database.add_to_discovery_pool(track_data, profile_id=profile_id): + if selected_source == 'spotify': + track_data['spotify_track_id'] = track['id'] + track_data['spotify_album_id'] = album_data['id'] + track_data['spotify_artist_id'] = selected_artist_id + elif selected_source == 'deezer': + track_data['deezer_track_id'] = track['id'] + track_data['deezer_album_id'] = album_data['id'] + track_data['deezer_artist_id'] = selected_artist_id + else: + track_data['itunes_track_id'] = track['id'] + track_data['itunes_album_id'] = album_data['id'] + track_data['itunes_artist_id'] = selected_artist_id + + if self.database.add_to_discovery_pool(track_data, source=selected_source, profile_id=profile_id): total_tracks_added += 1 except Exception as track_error: @@ -3593,8 +3626,8 @@ class WatchlistScanner: except Exception as e: logger.debug(f"Error building BYLT for {played_artist.get('name', '?')}: {e}") - # Also save without suffix for backward compatibility (use active source) - active_source = 'spotify' if spotify_available else fallback_source + # Also save without suffix for backward compatibility (use first active source). + active_source = sources_to_process[0] release_radar_key = f'release_radar_{active_source}' discovery_weekly_key = f'discovery_weekly_{active_source}' diff --git a/database/music_database.py b/database/music_database.py index 58c9baca..9cd3ca67 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -938,7 +938,7 @@ class MusicDatabase: """Add tables for discovery feature: similar artists, discovery pool, and recent releases""" try: # Similar Artists table - stores similar artists for each watchlist artist - # Supports both Spotify and iTunes IDs for dual-source discovery + # Supports Spotify plus fallback provider IDs for discovery cursor.execute(""" CREATE TABLE IF NOT EXISTS similar_artists ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -954,7 +954,7 @@ class MusicDatabase: """) # Discovery Pool table - rotating pool of 1000-2000 tracks for recommendations - # Supports both Spotify and iTunes sources for dual-source discovery + # Supports Spotify, iTunes, and Deezer sources for discovery cursor.execute(""" CREATE TABLE IF NOT EXISTS discovery_pool ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -980,7 +980,7 @@ class MusicDatabase: """) # Recent Releases table - tracks new releases from watchlist artists - # Supports both Spotify and iTunes sources for dual-source discovery + # Supports Spotify, iTunes, and Deezer sources for discovery cursor.execute(""" CREATE TABLE IF NOT EXISTS recent_releases ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -999,7 +999,7 @@ class MusicDatabase: """) # Discovery Recent Albums cache - for discover page recent releases section - # Supports both Spotify and iTunes sources for dual-source discovery + # Supports Spotify, iTunes, and Deezer sources for discovery cursor.execute(""" CREATE TABLE IF NOT EXISTS discovery_recent_albums ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -7162,7 +7162,7 @@ class MusicDatabase: return 0 def update_watchlist_artist_image(self, artist_id: str, image_url: str) -> bool: - """Update the image URL for a watchlist artist (checks both Spotify and iTunes IDs)""" + """Update the image URL for a watchlist artist (checks linked provider IDs)""" try: with self._get_connection() as conn: cursor = conn.cursor() @@ -7382,13 +7382,13 @@ class MusicDatabase: logger.error(f"Error getting similar artists: {e}") return [] - def get_similar_artists_missing_itunes_ids(self, source_artist_id: str, profile_id: int = 1) -> List[SimilarArtist]: - """Get similar artists for a source that are missing iTunes IDs (for backfill)""" - return self.get_similar_artists_missing_fallback_ids(source_artist_id, 'itunes', profile_id) - def get_similar_artists_missing_fallback_ids(self, source_artist_id: str, fallback_source: str = 'itunes', profile_id: int = 1) -> List[SimilarArtist]: - """Get similar artists missing IDs for the given fallback source (for backfill)""" + """Get similar artists missing fallback-provider IDs for backfill.""" try: + if fallback_source not in {'itunes', 'deezer'}: + logger.error("Unsupported similar-artist fallback source: %s", fallback_source) + return [] + col = 'similar_artist_deezer_id' if fallback_source == 'deezer' else 'similar_artist_itunes_id' with self._get_connection() as conn: cursor = conn.cursor() @@ -7499,19 +7499,16 @@ class MusicDatabase: logger.error(f"Error updating similar artist metadata by external ID: {e}") return False - def has_fresh_similar_artists(self, source_artist_id: str, days_threshold: int = 30, require_itunes: bool = True, require_spotify: bool = False, profile_id: int = 1) -> bool: + def has_fresh_similar_artists(self, source_artist_id: str, days_threshold: int = 30, profile_id: int = 1) -> bool: """ Check if we have cached similar artists that are still fresh (= days_threshold: return False - # Check if we have iTunes IDs (for seamless provider switching) - if require_itunes: - cursor.execute(""" - SELECT COUNT(*) as total, - SUM(CASE WHEN similar_artist_itunes_id IS NOT NULL AND similar_artist_itunes_id != '' THEN 1 ELSE 0 END) as has_itunes - FROM similar_artists - WHERE source_artist_id = ? AND profile_id = ? - """, (source_artist_id, profile_id)) - id_row = cursor.fetchone() - - if id_row and id_row['total'] > 0: - # If less than 50% have iTunes IDs, consider stale and refetch - itunes_ratio = id_row['has_itunes'] / id_row['total'] - if itunes_ratio < 0.5: - logger.debug(f"Similar artists for {source_artist_id} missing iTunes IDs ({id_row['has_itunes']}/{id_row['total']}), will refetch") - return False - - # Check if we have Spotify IDs (for Spotify discovery) - if require_spotify: - cursor.execute(""" - SELECT COUNT(*) as total, - SUM(CASE WHEN similar_artist_spotify_id IS NOT NULL AND similar_artist_spotify_id != '' THEN 1 ELSE 0 END) as has_spotify - FROM similar_artists - WHERE source_artist_id = ? AND profile_id = ? - """, (source_artist_id, profile_id)) - id_row = cursor.fetchone() - - if id_row and id_row['total'] > 0: - # If less than 50% have Spotify IDs, consider stale and refetch - spotify_ratio = id_row['has_spotify'] / id_row['total'] - if spotify_ratio < 0.5: - logger.debug(f"Similar artists for {source_artist_id} missing Spotify IDs ({id_row['has_spotify']}/{id_row['total']}), will refetch") - return False - return True except Exception as e: @@ -7669,7 +7632,7 @@ class MusicDatabase: logger.error(f"Error marking artists as featured: {e}") def add_to_discovery_pool(self, track_data: Dict[str, Any], source: str = 'spotify', profile_id: int = 1) -> bool: - """Add a track to the discovery pool (supports both Spotify and iTunes sources)""" + """Add a track to the discovery pool (supports Spotify, iTunes, and Deezer sources)""" try: with self._get_connection() as conn: cursor = conn.cursor() @@ -7764,7 +7727,7 @@ class MusicDatabase: logger.error(f"Error rotating discovery pool: {e}") def get_discovery_pool_tracks(self, limit: int = 100, new_releases_only: bool = False, source: Optional[str] = None, profile_id: int = 1) -> List[DiscoveryTrack]: - """Get tracks from discovery pool, optionally filtered by source ('spotify' or 'itunes')""" + """Get tracks from discovery pool, optionally filtered by source ('spotify', 'itunes', or 'deezer')""" try: with self._get_connection() as conn: cursor = conn.cursor() @@ -7822,7 +7785,7 @@ class MusicDatabase: return [] def cache_discovery_recent_album(self, album_data: Dict[str, Any], source: str = 'spotify', profile_id: int = 1) -> bool: - """Cache a recent album for the discover page (supports both Spotify and iTunes sources)""" + """Cache a recent album for the discover page (supports Spotify, iTunes, and Deezer sources)""" try: with self._get_connection() as conn: cursor = conn.cursor() diff --git a/tests/test_watchlist_scanner_scan.py b/tests/test_watchlist_scanner_scan.py index a1fad3bd..85d8d607 100644 --- a/tests/test_watchlist_scanner_scan.py +++ b/tests/test_watchlist_scanner_scan.py @@ -220,6 +220,130 @@ def _build_scanner(album_data, artists): return scanner +def test_fetch_similar_artists_from_musicmap_uses_provider_priority(monkeypatch): + html = """ + + +
+ Artist One + Similar Artist +
+ + + """ + + class _Response: + def __init__(self, text): + self.text = text + + def raise_for_status(self): + return None + + def make_client(source, seed_id, match_id, canonical_name, popularity): + client = _FakeSourceClient(artist_id=match_id, albums=[], image_url=None) + + def search_artists(query, limit=1, **kwargs): + client.search_calls.append((query, limit, kwargs)) + if query == "Artist One": + return [types.SimpleNamespace(id=seed_id, name=f"{source} Seed")] + if query == "Similar Artist": + return [ + types.SimpleNamespace( + id=match_id, + name=canonical_name, + image_url=f"https://{source}.example.com/{match_id}.jpg", + genres=[source, "genre"], + popularity=popularity, + ) + ] + return [] + + client.search_artists = search_artists + return client + + deezer_client = make_client("deezer", "dz-seed", "dz-match", "Deezer Canonical", 30) + itunes_client = make_client("itunes", "it-seed", "it-match", "iTunes Canonical", 20) + spotify_client = make_client("spotify", "sp-seed", "sp-match", "Spotify Canonical", 10) + + monkeypatch.setattr(watchlist_scanner_module.requests, "get", lambda *args, **kwargs: _Response(html)) + monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer") + monkeypatch.setattr( + watchlist_scanner_module, + "get_client_for_source", + lambda source: { + "deezer": deezer_client, + "itunes": itunes_client, + "spotify": spotify_client, + }.get(source), + ) + + scanner = _build_scanner({"tracks": {"items": []}}, []) + results = scanner._fetch_similar_artists_from_musicmap("Artist One", limit=5) + + assert len(results) == 1 + artist = results[0] + assert artist["name"] == "Deezer Canonical" + assert artist["deezer_id"] == "dz-match" + assert artist["itunes_id"] == "it-match" + assert artist["spotify_id"] == "sp-match" + assert artist["image_url"] == "https://deezer.example.com/dz-match.jpg" + assert artist["genres"] == ["deezer", "genre"] + assert artist["popularity"] == 30 + + assert [call[0] for call in deezer_client.search_calls] == ["Artist One", "Similar Artist"] + assert [call[0] for call in itunes_client.search_calls] == ["Artist One", "Similar Artist"] + assert [call[0] for call in spotify_client.search_calls] == ["Artist One", "Similar Artist"] + assert spotify_client.search_calls[-1][2]["allow_fallback"] is False + + +def test_backfill_similar_artists_fallback_ids_uses_provider_priority(monkeypatch): + def make_client(source): + client = types.SimpleNamespace(search_calls=[]) + + def search_artists(query, limit=1, **kwargs): + client.search_calls.append((query, limit, kwargs)) + safe_name = query.lower().replace(" ", "-") + return [types.SimpleNamespace(id=f"{source}-{safe_name}", name=query)] + + client.search_artists = search_artists + return client + + deezer_client = make_client("deezer") + itunes_client = make_client("itunes") + + deezer_artist = types.SimpleNamespace(id=11, similar_artist_name="Deezer Artist") + itunes_artist = types.SimpleNamespace(id=22, similar_artist_name="iTunes Artist") + + monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer") + monkeypatch.setattr( + watchlist_scanner_module, + "get_client_for_source", + lambda source: { + "deezer": deezer_client, + "itunes": itunes_client, + }.get(source), + ) + + scanner = _build_scanner({"tracks": {"items": []}}, []) + scanner.database.get_similar_artists_missing_fallback_ids = ( + lambda source_artist_id, fallback_source, profile_id=1: [deezer_artist] if fallback_source == "deezer" else [itunes_artist] + ) + + update_calls = [] + scanner.database.update_similar_artist_deezer_id = lambda similar_artist_id, deezer_id: update_calls.append(("deezer", similar_artist_id, deezer_id)) or True + scanner.database.update_similar_artist_itunes_id = lambda similar_artist_id, itunes_id: update_calls.append(("itunes", similar_artist_id, itunes_id)) or True + + count = scanner._backfill_similar_artists_fallback_ids("source-artist", profile_id=7) + + assert count == 2 + assert update_calls == [ + ("deezer", 11, "deezer-deezer-artist"), + ("itunes", 22, "itunes-itunes-artist"), + ] + assert [call[0] for call in deezer_client.search_calls] == ["Deezer Artist"] + assert [call[0] for call in itunes_client.search_calls] == ["iTunes Artist"] + + def test_scan_watchlist_profile_loads_artists_and_applies_overrides(monkeypatch): artist = _build_artist() scanner = _build_scanner({"tracks": {"items": []}}, [artist]) @@ -276,7 +400,7 @@ def test_scan_watchlist_artists_scans_tracks_and_updates_state(monkeypatch): monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *_args, **_kwargs: True) monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True) monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True) - monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0) + monkeypatch.setattr(scanner, "_backfill_similar_artists_fallback_ids", lambda *_args, **_kwargs: 0) scan_state = {} results = scanner.scan_watchlist_artists([artist], scan_state=scan_state) @@ -336,7 +460,7 @@ def test_scan_watchlist_artists_skips_placeholder_tracklists(monkeypatch): monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *args, **kwargs: add_calls.append((args, kwargs)) or True) monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True) monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True) - monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0) + monkeypatch.setattr(scanner, "_backfill_similar_artists_fallback_ids", lambda *_args, **_kwargs: 0) scan_state = {} results = scanner.scan_watchlist_artists([artist], scan_state=scan_state) @@ -375,7 +499,7 @@ def test_scan_watchlist_artists_honors_cancel_check(monkeypatch): monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *_args, **_kwargs: False) monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True) monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True) - monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0) + monkeypatch.setattr(scanner, "_backfill_similar_artists_fallback_ids", lambda *_args, **_kwargs: 0) cancels = iter([False, True]) scan_state = {} @@ -827,6 +951,74 @@ def test_cache_discovery_recent_albums_falls_back_to_spotify_when_primary_has_no assert spotify_client.album_calls +def test_update_discovery_pool_incremental_uses_source_priority(monkeypatch): + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0) + monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None)) + monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + + artist = _build_artist("Incremental Artist") + artist.spotify_artist_id = None + artist.deezer_artist_id = None + + release = types.SimpleNamespace( + id="dz-release-1", + name="Incremental Release", + release_date="2026-04-16", + album_type="album", + image_url="https://example.com/deezer-release.jpg", + ) + + deezer_client = _FakeSourceClient( + artist_id="dz-artist", + albums=[release], + image_url="https://example.com/deezer-artist.jpg", + album_payload={ + "id": "dz-release-1", + "name": "Incremental Release", + "images": [{"url": "https://example.com/deezer-release.jpg"}], + "release_date": "2026-04-16", + "popularity": 10, + "tracks": {"items": [{"id": "dz-track-1", "name": "Incremental Track", "artists": [{"name": "Incremental Artist"}], "duration_ms": 180000}]}, + "artists": [{"id": "dz-artist"}], + }, + ) + spotify_client = _FakeSourceClient( + artist_id="sp-artist", + albums=[], + image_url="https://example.com/spotify-artist.jpg", + album_payload={ + "id": "sp-release-1", + "name": "Spotify Incremental Release", + "images": [{"url": "https://example.com/spotify-release.jpg"}], + "release_date": "2026-04-16", + "popularity": 50, + "tracks": {"items": [{"id": "sp-track-1", "name": "Spotify Incremental Track", "artists": [{"name": "Incremental Artist"}], "duration_ms": 180000}]}, + "artists": [{"id": "sp-artist"}], + }, + ) + + def fake_get_client_for_source(source): + return { + "deezer": deezer_client, + "spotify": spotify_client, + }.get(source) + + monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source) + + scanner = _build_scanner({"tracks": {"items": []}}, [artist]) + scanner.database.should_populate_discovery_pool = lambda hours_threshold=6, profile_id=1: True + + scanner.update_discovery_pool_incremental(profile_id=1) + + assert scanner.database.discovery_pool_calls + assert scanner.database.discovery_pool_calls[0][1] == "deezer" + assert deezer_client.search_calls == [("Incremental Artist", 1, {})] + assert deezer_client.album_calls + assert spotify_client.search_calls == [] + assert spotify_client.album_calls == [] + + def test_curate_discovery_playlists_uses_source_priority_for_recent_albums(monkeypatch): monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0) monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer") @@ -913,6 +1105,29 @@ def test_curate_discovery_playlists_uses_source_priority_for_recent_albums(monke assert any(key == "discovery_weekly_deezer" for key, _ in saved_playlists) +def test_has_fresh_similar_artists_uses_age_only(tmp_path): + from datetime import datetime + from database.music_database import MusicDatabase + + db = MusicDatabase(str(tmp_path / "music.db")) + db.add_or_update_similar_artist( + source_artist_id="source-1", + similar_artist_name="Similar Artist", + similar_artist_itunes_id="it-artist", + similar_artist_deezer_id="dz-artist", + profile_id=1, + ) + + with db._get_connection() as conn: + conn.execute( + "UPDATE similar_artists SET last_updated = ? WHERE source_artist_id = ? AND profile_id = ?", + (datetime.now().isoformat(), "source-1", 1), + ) + conn.commit() + + assert db.has_fresh_similar_artists("source-1", days_threshold=30, profile_id=1) is True + + def test_match_to_spotify_uses_strict_lookup(): spotify_client = _FakeSpotifyClient( search_results=[types.SimpleNamespace(id="fallback-id", name="Artist One")]