From f126cf7118e007df84021ed6494a66617c90b6b5 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Thu, 22 Jan 2026 17:04:23 -0800 Subject: [PATCH 01/20] Add cross-provider support for watchlist artists Introduces iTunes artist ID support to WatchlistArtist and database schema, enabling proactive backfilling of missing provider IDs (Spotify/iTunes) for watchlist artists. Updates WatchlistScanner to use MetadataService for provider-agnostic scanning and ID matching, and modifies web_server to support scans with either provider. Includes new database migration and update methods for iTunes and Spotify artist IDs. --- core/watchlist_scanner.py | 117 ++++++++++++++++++++++++++++++++++++- database/music_database.py | 66 ++++++++++++++++++++- web_server.py | 35 +++++++++-- 3 files changed, 209 insertions(+), 9 deletions(-) diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 3d25872e..d85baec9 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -232,12 +232,21 @@ class ScanResult: class WatchlistScanner: """Service for scanning watched artists for new releases""" - def __init__(self, spotify_client: SpotifyClient, database_path: str = "database/music_library.db"): - self.spotify_client = spotify_client + def __init__(self, spotify_client: SpotifyClient = None, metadata_service=None, database_path: str = "database/music_library.db"): + # Support both old (spotify_client) and new (metadata_service) initialization self.database_path = database_path self._database = None self._wishlist_service = None self._matching_engine = None + + if metadata_service: + self._metadata_service = metadata_service + self.spotify_client = metadata_service.spotify # For backward compatibility + elif spotify_client: + self.spotify_client = spotify_client + self._metadata_service = None # Lazy load if needed + else: + raise ValueError("Must provide either spotify_client or metadata_service") @property def database(self): @@ -260,6 +269,14 @@ class WatchlistScanner: self._matching_engine = MusicMatchingEngine() return self._matching_engine + @property + def metadata_service(self): + """Get or create MetadataService instance (lazy loading)""" + if self._metadata_service is None: + from core.metadata_service import MetadataService + self._metadata_service = MetadataService() + return self._metadata_service + def scan_all_watchlist_artists(self) -> List[ScanResult]: """ Scan artists in the watchlist for new releases. @@ -326,6 +343,22 @@ class WatchlistScanner: watchlist_artists = artists_to_scan + # PROACTIVE ID BACKFILLING (cross-provider support) + # Before scanning, ensure all artists have IDs for the current provider + logger.info(f"DEBUG: About to check backfilling. _metadata_service = {getattr(self, '_metadata_service', 'ATTRIBUTE MISSING')}") + if self._metadata_service is not None: + try: + active_provider = self._metadata_service.get_active_provider() + logger.info(f"πŸ” Checking for missing {active_provider} IDs in watchlist...") + self._backfill_missing_ids(all_watchlist_artists, active_provider) + except Exception as backfill_error: + logger.warning(f"Error during ID backfilling: {backfill_error}") + import traceback + traceback.print_exc() + # Continue with scan even if backfilling fails + else: + logger.warning(f"⚠️ Backfilling SKIPPED - _metadata_service is None") + scan_results = [] for i, artist in enumerate(watchlist_artists): try: @@ -559,6 +592,86 @@ class WatchlistScanner: logger.error(f"Error getting discography for artist {spotify_artist_id}: {e}") return None + def _backfill_missing_ids(self, artists: List[WatchlistArtist], provider: str): + """ + Proactively match ALL artists missing IDs for the current provider. + + Example: User has 50 artists with only Spotify IDs. + When iTunes becomes active, this matches ALL 50 to iTunes in one batch. + """ + artists_to_match = [] + + if provider == 'spotify': + # Find all artists missing Spotify IDs + artists_to_match = [a for a in artists if not a.spotify_artist_id and a.itunes_artist_id] + elif provider == 'itunes': + # Find all artists missing iTunes IDs + artists_to_match = [a for a in artists if not a.itunes_artist_id and a.spotify_artist_id] + + if not artists_to_match: + logger.info(f"βœ… All artists already have {provider} IDs") + return + + logger.info(f"πŸ”„ Backfilling {len(artists_to_match)} artists with {provider} IDs...") + + matched_count = 0 + for artist in artists_to_match: + try: + if provider == 'spotify': + new_id = self._match_to_spotify(artist.artist_name) + if new_id: + self.database.update_watchlist_spotify_id(artist.id, new_id) + artist.spotify_artist_id = new_id # Update in memory + matched_count += 1 + logger.info(f"βœ… Matched '{artist.artist_name}' to Spotify: {new_id}") + + elif provider == 'itunes': + new_id = self._match_to_itunes(artist.artist_name) + if new_id: + self.database.update_watchlist_itunes_id(artist.id, new_id) + artist.itunes_artist_id = new_id # Update in memory + matched_count += 1 + logger.info(f"βœ… Matched '{artist.artist_name}' to iTunes: {new_id}") + + # Small delay to avoid API rate limits + time.sleep(0.3) + + except Exception as e: + logger.warning(f"Could not match '{artist.artist_name}' to {provider}: {e}") + continue + + logger.info(f"βœ… Backfilled {matched_count}/{len(artists_to_match)} artists with {provider} IDs") + + def _match_to_spotify(self, artist_name: str) -> Optional[str]: + """Match artist name to Spotify ID""" + try: + # Use metadata service if available, fallback to spotify_client + if hasattr(self, '_metadata_service') and self._metadata_service: + results = self._metadata_service.spotify.search_artists(artist_name, limit=1) + else: + results = self.spotify_client.search_artists(artist_name, limit=1) + + if results: + return results[0].id + except Exception as e: + logger.warning(f"Could not match {artist_name} to Spotify: {e}") + return None + + def _match_to_itunes(self, artist_name: str) -> Optional[str]: + """Match artist name to iTunes ID""" + try: + # Use metadata service's iTunes client + if hasattr(self, '_metadata_service') and self._metadata_service: + results = self._metadata_service.itunes.search_artists(artist_name, limit=1) + if results: + return results[0].id + else: + # iTunes client not available without metadata service + logger.warning(f"Cannot match to iTunes - MetadataService not available") + except Exception as e: + logger.warning(f"Could not match {artist_name} to iTunes: {e}") + return None + def _get_lookback_period_setting(self) -> str: """ Get the discovery lookback period setting from database. diff --git a/database/music_database.py b/database/music_database.py index b70d0617..71f612cb 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -79,13 +79,14 @@ class DatabaseTrackWithMetadata: class WatchlistArtist: """Artist being monitored for new releases""" id: int - spotify_artist_id: str + spotify_artist_id: Optional[str] # Can be None if added via iTunes artist_name: str date_added: datetime last_scan_timestamp: Optional[datetime] = None created_at: Optional[datetime] = None updated_at: Optional[datetime] = None image_url: Optional[str] = None + itunes_artist_id: Optional[str] = None # Cross-provider support include_albums: bool = True include_eps: bool = True include_singles: bool = True @@ -280,6 +281,9 @@ class MusicDatabase: # Add content type filter columns to watchlist_artists (migration) self._add_watchlist_content_type_filters(cursor) + # Add iTunes artist ID column to watchlist_artists (migration) + self._add_watchlist_itunes_id_column(cursor) + conn.commit() logger.info("Database initialized successfully") @@ -637,7 +641,7 @@ class MusicDatabase: columns = [column[1] for column in cursor.fetchall()] columns_to_add = { - 'include_live': ('INTEGER', '0'), # 0 = False (exclude live versions by default) + 'include_live': ('INTEGER', '0'), # 0 = False (exclude live versions by default) 'include_remixes': ('INTEGER', '0'), # 0 = False (exclude remixes by default) 'include_acoustic': ('INTEGER', '0'), # 0 = False (exclude acoustic by default) 'include_compilations': ('INTEGER', '0') # 0 = False (exclude compilations by default) @@ -652,6 +656,20 @@ class MusicDatabase: logger.error(f"Error adding content type filter columns to watchlist_artists: {e}") # Don't raise - this is a migration, database can still function + def _add_watchlist_itunes_id_column(self, cursor): + """Add iTunes artist ID column to watchlist_artists table for cross-provider support""" + try: + cursor.execute("PRAGMA table_info(watchlist_artists)") + columns = [column[1] for column in cursor.fetchall()] + + if 'itunes_artist_id' not in columns: + cursor.execute("ALTER TABLE watchlist_artists ADD COLUMN itunes_artist_id TEXT") + logger.info("Added itunes_artist_id column to watchlist_artists table for cross-provider support") + + except Exception as e: + logger.error(f"Error adding itunes_artist_id column to watchlist_artists: {e}") + # Don't raise - this is a migration, database can still function + def close(self): """Close database connection (no-op since we create connections per operation)""" # Each operation creates and closes its own connection, so nothing to do here @@ -2755,7 +2773,7 @@ class MusicDatabase: # Build SELECT query based on existing columns base_columns = ['id', 'spotify_artist_id', 'artist_name', 'date_added', 'last_scan_timestamp', 'created_at', 'updated_at'] - optional_columns = ['image_url', 'include_albums', 'include_eps', 'include_singles', + optional_columns = ['image_url', 'itunes_artist_id', 'include_albums', 'include_eps', 'include_singles', 'include_live', 'include_remixes', 'include_acoustic', 'include_compilations'] columns_to_select = base_columns + [col for col in optional_columns if col in existing_columns] @@ -2772,6 +2790,7 @@ class MusicDatabase: for row in rows: # Safely get optional columns with defaults (sqlite3.Row uses dict-style access) image_url = row['image_url'] if 'image_url' in existing_columns else None + itunes_artist_id = row['itunes_artist_id'] if 'itunes_artist_id' in existing_columns else None include_albums = bool(row['include_albums']) if 'include_albums' in existing_columns else True include_eps = bool(row['include_eps']) if 'include_eps' in existing_columns else True include_singles = bool(row['include_singles']) if 'include_singles' in existing_columns else True @@ -2789,6 +2808,7 @@ class MusicDatabase: created_at=datetime.fromisoformat(row['created_at']) if row['created_at'] else None, updated_at=datetime.fromisoformat(row['updated_at']) if row['updated_at'] else None, image_url=image_url, + itunes_artist_id=itunes_artist_id, include_albums=include_albums, include_eps=include_eps, include_singles=include_singles, @@ -2846,6 +2866,46 @@ class MusicDatabase: logger.error(f"Error updating watchlist artist image: {e}") return False + def update_watchlist_spotify_id(self, watchlist_id: int, spotify_id: str) -> bool: + """Update the Spotify artist ID for a watchlist artist (cross-provider support)""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute(""" + UPDATE watchlist_artists + SET spotify_artist_id = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, (spotify_id, watchlist_id)) + + conn.commit() + logger.info(f"Updated Spotify ID for watchlist artist {watchlist_id}: {spotify_id}") + return cursor.rowcount > 0 + + except Exception as e: + logger.error(f"Error updating watchlist Spotify ID: {e}") + return False + + def update_watchlist_itunes_id(self, watchlist_id: int, itunes_id: str) -> bool: + """Update the iTunes artist ID for a watchlist artist (cross-provider support)""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute(""" + UPDATE watchlist_artists + SET itunes_artist_id = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, (itunes_id, watchlist_id)) + + conn.commit() + logger.info(f"Updated iTunes ID for watchlist artist {watchlist_id}: {itunes_id}") + return cursor.rowcount > 0 + + except Exception as e: + logger.error(f"Error updating watchlist iTunes ID: {e}") + return False + # === Discovery Feature Methods === def add_or_update_similar_artist(self, source_artist_id: str, similar_artist_spotify_id: str, diff --git a/web_server.py b/web_server.py index 2bcad806..d83bb7b4 100644 --- a/web_server.py +++ b/web_server.py @@ -17093,8 +17093,22 @@ def check_watchlist_status(): def start_watchlist_scan(): """Start a watchlist scan for new releases""" try: - if not spotify_client or not spotify_client.is_authenticated(): - return jsonify({"success": False, "error": "Spotify client not available or not authenticated"}), 400 + # Check if MetadataService can provide a working client (Spotify OR iTunes) + from core.metadata_service import MetadataService + metadata_service = MetadataService() + + # Get active provider - will be either spotify or itunes + active_provider = metadata_service.get_active_provider() + provider_info = metadata_service.get_provider_info() + + # Verify we have at least one working provider + if not provider_info['spotify_authenticated'] and not provider_info['itunes_available']: + return jsonify({ + "success": False, + "error": "No music provider available. Please authenticate Spotify or ensure iTunes is accessible." + }), 400 + + logger.info(f"Starting watchlist scan with {active_provider} provider") # Check if wishlist auto-processing is currently running (using smart detection) if is_wishlist_actually_processing(): @@ -17108,7 +17122,7 @@ def start_watchlist_scan(): def run_scan(): try: global watchlist_scan_state, watchlist_auto_scanning, watchlist_auto_scanning_timestamp - from core.watchlist_scanner import get_watchlist_scanner + from core.watchlist_scanner import WatchlistScanner from database.music_database import get_database # Set flag and timestamp for manual scan @@ -17137,7 +17151,20 @@ def start_watchlist_scan(): watchlist_next_run_time = 0 # Clear timer for consistency return - scanner = get_watchlist_scanner(spotify_client) + # Initialize scanner with MetadataService for cross-provider support + scanner = WatchlistScanner(metadata_service=metadata_service) + + # PROACTIVE ID BACKFILLING (cross-provider support) + # Before scanning, ensure all artists have IDs for the current provider + try: + active_provider = metadata_service.get_active_provider() + print(f"πŸ” Checking for missing {active_provider} IDs in watchlist...") + scanner._backfill_missing_ids(watchlist_artists, active_provider) + except Exception as backfill_error: + print(f"⚠️ Error during ID backfilling: {backfill_error}") + import traceback + traceback.print_exc() + # Continue with scan even if backfilling fails # Initialize detailed progress tracking watchlist_scan_state.update({ From 579124c477e31e7d83b65fd79afc2085ab240f0b Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Thu, 22 Jan 2026 17:32:23 -0800 Subject: [PATCH 02/20] Normalize iTunes API responses to Spotify format Updated get_album, get_album_tracks, and get_artist methods to return data structures compatible with Spotify's API format. This includes normalizing image arrays, artist and album fields, and adding synthetic URIs and external URLs for better interoperability. --- core/itunes_client.py | 114 +++++++++++++++++++++++++++++++++++------- 1 file changed, 95 insertions(+), 19 deletions(-) diff --git a/core/itunes_client.py b/core/itunes_client.py index bc24bf99..7f430d0a 100644 --- a/core/itunes_client.py +++ b/core/itunes_client.py @@ -347,34 +347,83 @@ class iTunesClient: return albums def get_album(self, album_id: str) -> Optional[Dict[str, Any]]: - """Get album information""" + """Get album information - normalized to Spotify format""" results = self._lookup(id=album_id) - + for album_data in results: if album_data.get('wrapperType') == 'collection': - return album_data - + # Normalize to Spotify-compatible format + image_url = None + if album_data.get('artworkUrl100'): + image_url = album_data['artworkUrl100'].replace('100x100bb', '600x600bb') + + # Build images array like Spotify (multiple sizes) + images = [] + if image_url: + images = [ + {'url': image_url, 'height': 600, 'width': 600}, + {'url': album_data['artworkUrl100'].replace('100x100bb', '300x300bb'), 'height': 300, 'width': 300}, + {'url': album_data['artworkUrl100'], 'height': 100, 'width': 100} + ] + + # Determine album type + track_count = album_data.get('trackCount', 0) + if track_count <= 3: + album_type = 'single' + elif track_count <= 6: + album_type = 'single' # EP treated as single + else: + album_type = 'album' + + return { + 'id': str(album_data.get('collectionId', '')), + 'name': album_data.get('collectionName', ''), + 'images': images, + 'artists': [{'name': album_data.get('artistName', 'Unknown Artist'), 'id': str(album_data.get('artistId', ''))}], + 'release_date': album_data.get('releaseDate', '')[:10] if album_data.get('releaseDate') else '', # YYYY-MM-DD format + 'total_tracks': track_count, + 'album_type': album_type, + 'external_urls': {'itunes': album_data.get('collectionViewUrl', '')}, + 'uri': f"itunes:album:{album_data.get('collectionId', '')}", + '_source': 'itunes', + '_raw_data': album_data + } + return None def get_album_tracks(self, album_id: str) -> Optional[Dict[str, Any]]: - """Get album tracks with all tracks included""" + """Get album tracks - normalized to Spotify format""" results = self._lookup(id=album_id, entity='song') - + if not results: return None - + # First result is usually the album/collection info # Remaining results are tracks tracks = [] for item in results: if item.get('wrapperType') == 'track' and item.get('kind') == 'song': - tracks.append(item) - + # Normalize each track to Spotify-compatible format + normalized_track = { + 'id': str(item.get('trackId', '')), + 'name': item.get('trackName', ''), + 'artists': [{'name': item.get('artistName', 'Unknown Artist')}], # List of dicts like Spotify + 'duration_ms': item.get('trackTimeMillis', 0), + 'track_number': item.get('trackNumber', 0), + 'disc_number': item.get('discNumber', 1), + 'explicit': item.get('trackExplicitness') == 'explicit', + 'preview_url': item.get('previewUrl'), + 'uri': f"itunes:track:{item.get('trackId', '')}", # Synthetic URI + 'external_urls': {'itunes': item.get('trackViewUrl', '')}, + '_source': 'itunes' + } + tracks.append(normalized_track) + # Sort by disc and track number - tracks.sort(key=lambda t: (t.get('discNumber', 1), t.get('trackNumber', 0))) - + tracks.sort(key=lambda t: (t.get('disc_number', 1), t.get('track_number', 0))) + logger.info(f"Retrieved {len(tracks)} tracks for album {album_id}") - + return { 'items': tracks, 'total': len(tracks), @@ -399,20 +448,47 @@ class iTunesClient: def get_artist(self, artist_id: str) -> Optional[Dict[str, Any]]: """ - Get full artist details from iTunes API. - + Get full artist details - normalized to Spotify format. + Args: artist_id: iTunes artist ID - + Returns: - Dictionary with artist data + Dictionary with artist data matching Spotify's format """ results = self._lookup(id=artist_id) - + for artist_data in results: if artist_data.get('wrapperType') == 'artist': - return artist_data - + # Build images array - iTunes artist search doesn't reliably return images + # but we include the structure for compatibility + images = [] + if artist_data.get('artworkUrl100'): + artwork_base = artist_data['artworkUrl100'] + images = [ + {'url': artwork_base.replace('100x100bb', '600x600bb'), 'height': 600, 'width': 600}, + {'url': artwork_base.replace('100x100bb', '300x300bb'), 'height': 300, 'width': 300}, + {'url': artwork_base, 'height': 100, 'width': 100} + ] + + # Get genre + genres = [] + if artist_data.get('primaryGenreName'): + genres = [artist_data['primaryGenreName']] + + return { + 'id': str(artist_data.get('artistId', '')), + 'name': artist_data.get('artistName', ''), + 'images': images, + 'genres': genres, + 'popularity': 0, # iTunes doesn't provide this + 'followers': {'total': 0}, # iTunes doesn't provide this + 'external_urls': {'itunes': artist_data.get('artistViewUrl', '')}, + 'uri': f"itunes:artist:{artist_data.get('artistId', '')}", + '_source': 'itunes', + '_raw_data': artist_data + } + return None def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 50) -> List[Album]: From b790c346578af9fd11d6b6e69083dd14bfe2759f Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Thu, 22 Jan 2026 18:14:23 -0800 Subject: [PATCH 03/20] Add support for scanning artists with iTunes provider Refactored artist scanning logic to use the active metadata provider (Spotify or iTunes) for fetching artist data, discography, and album tracks. Introduced helper methods to select the correct client and artist ID based on the provider, and updated image and similar artist handling accordingly. This enables watchlist scanning to work with iTunes when Spotify is not authenticated, improving flexibility and provider support. --- core/watchlist_scanner.py | 166 ++++++++++++++++++++++++++++++-------- 1 file changed, 134 insertions(+), 32 deletions(-) diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index d85baec9..d79962f3 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -276,6 +276,28 @@ class WatchlistScanner: from core.metadata_service import MetadataService self._metadata_service = MetadataService() return self._metadata_service + + def _get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist): + """ + Get the appropriate client and artist ID based on active provider. + + Returns: + Tuple of (client, artist_id, provider_name) or (None, None, None) if no valid ID + """ + provider = self.metadata_service.get_active_provider() + + if provider == 'spotify': + if watchlist_artist.spotify_artist_id: + return (self.metadata_service.spotify, watchlist_artist.spotify_artist_id, 'spotify') + else: + logger.warning(f"No Spotify ID for {watchlist_artist.artist_name}, cannot scan with Spotify") + return (None, None, None) + else: # itunes + if watchlist_artist.itunes_artist_id: + return (self.metadata_service.itunes, watchlist_artist.itunes_artist_id, 'itunes') + else: + logger.warning(f"No iTunes ID for {watchlist_artist.artist_name}, cannot scan with iTunes") + return (None, None, None) def scan_all_watchlist_artists(self) -> List[ScanResult]: """ @@ -414,13 +436,30 @@ class WatchlistScanner: """ Scan a single artist for new releases. Only checks releases after the last scan timestamp. + Uses the active provider (Spotify if authenticated, otherwise iTunes). """ try: logger.info(f"Scanning artist: {watchlist_artist.artist_name}") - # Update artist image from Spotify (cached for performance) + # Get the active client and artist ID based on provider + client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist) + + if client is None or artist_id is None: + return ScanResult( + artist_name=watchlist_artist.artist_name, + spotify_artist_id=watchlist_artist.spotify_artist_id or '', + albums_checked=0, + new_tracks_found=0, + tracks_added_to_wishlist=0, + success=False, + error_message=f"No {self.metadata_service.get_active_provider()} ID available for this artist" + ) + + logger.info(f"Using {provider} provider for {watchlist_artist.artist_name} (ID: {artist_id})") + + # Update artist image (cached for performance) try: - artist_data = self.spotify_client.get_artist(watchlist_artist.spotify_artist_id) + artist_data = client.get_artist(artist_id) if artist_data and 'images' in artist_data and artist_data['images']: # Get medium-sized image (usually the second one, or first if only one) image_url = None @@ -429,52 +468,59 @@ class WatchlistScanner: else: image_url = artist_data['images'][0]['url'] - # Update in database + # Update in database (use spotify_artist_id as the key for consistency) if image_url: - self.database.update_watchlist_artist_image(watchlist_artist.spotify_artist_id, image_url) + db_artist_id = watchlist_artist.spotify_artist_id or artist_id + self.database.update_watchlist_artist_image(db_artist_id, image_url) logger.info(f"Updated artist image for {watchlist_artist.artist_name}") else: logger.warning(f"No image URL found for {watchlist_artist.artist_name}") else: - logger.warning(f"No images in Spotify data for {watchlist_artist.artist_name}") + logger.warning(f"No images in {provider} data for {watchlist_artist.artist_name}") except Exception as img_error: logger.warning(f"Could not update artist image for {watchlist_artist.artist_name}: {img_error}") - # Get artist discography from Spotify - albums = self.get_artist_discography(watchlist_artist.spotify_artist_id, watchlist_artist.last_scan_timestamp) + # Get artist discography using active provider + albums = self._get_artist_discography_with_client(client, artist_id, watchlist_artist.last_scan_timestamp) if albums is None: return ScanResult( artist_name=watchlist_artist.artist_name, - spotify_artist_id=watchlist_artist.spotify_artist_id, + spotify_artist_id=watchlist_artist.spotify_artist_id or '', albums_checked=0, new_tracks_found=0, tracks_added_to_wishlist=0, success=False, - error_message="Failed to get artist discography from Spotify" + error_message=f"Failed to get artist discography from {provider}" ) - + logger.info(f"Found {len(albums)} albums/singles to check for {watchlist_artist.artist_name}") - + # Safety check: Limit number of albums to scan to prevent extremely long sessions MAX_ALBUMS_PER_ARTIST = 50 # Reasonable limit to prevent API abuse if len(albums) > MAX_ALBUMS_PER_ARTIST: logger.warning(f"Artist {watchlist_artist.artist_name} has {len(albums)} albums, limiting to {MAX_ALBUMS_PER_ARTIST} most recent") albums = albums[:MAX_ALBUMS_PER_ARTIST] # Most recent albums are first - + # Check each album/single for missing tracks new_tracks_found = 0 tracks_added_to_wishlist = 0 - + for album_index, album in enumerate(albums): try: - # Get full album data with tracks + # Get full album data logger.info(f"Checking album {album_index + 1}/{len(albums)}: {album.name}") - album_data = self.spotify_client.get_album(album.id) - if not album_data or 'tracks' not in album_data or not album_data['tracks'].get('items'): + album_data = client.get_album(album.id) + if not album_data: continue - - tracks = album_data['tracks']['items'] + + # Get album tracks (works for both Spotify and iTunes) + # Spotify's get_album() includes tracks, but we use get_album_tracks() for consistency + tracks_data = client.get_album_tracks(album.id) + if not tracks_data or not tracks_data.get('items'): + continue + + tracks = tracks_data['items'] logger.debug(f"Checking album: {album_data.get('name', 'Unknown')} ({len(tracks)} tracks)") # Check if user wants this type of release @@ -506,24 +552,29 @@ class WatchlistScanner: logger.warning(f"Error checking album {album.name}: {e}") continue - # Update last scan timestamp for this artist - self.update_artist_scan_timestamp(watchlist_artist.spotify_artist_id) + # Update last scan timestamp for this artist (use spotify_artist_id as DB key for consistency) + db_artist_id = watchlist_artist.spotify_artist_id or artist_id + self.update_artist_scan_timestamp(db_artist_id) # Fetch and store similar artists for discovery feature (with caching to avoid over-polling) - try: - # Check if we have fresh similar artists cached (< 30 days old) - if self.database.has_fresh_similar_artists(watchlist_artist.spotify_artist_id, days_threshold=30): - logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping fetch") - else: - logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}...") - self.update_similar_artists(watchlist_artist) - logger.info(f"Similar artists updated for {watchlist_artist.artist_name}") - except Exception as similar_error: - logger.warning(f"Failed to update similar artists for {watchlist_artist.artist_name}: {similar_error}") + # Note: Similar artists feature only works with Spotify + if provider == 'spotify' and watchlist_artist.spotify_artist_id: + try: + # Check if we have fresh similar artists cached (< 30 days old) + if self.database.has_fresh_similar_artists(watchlist_artist.spotify_artist_id, days_threshold=30): + logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping fetch") + else: + logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}...") + self.update_similar_artists(watchlist_artist) + logger.info(f"Similar artists updated for {watchlist_artist.artist_name}") + except Exception as similar_error: + logger.warning(f"Failed to update similar artists for {watchlist_artist.artist_name}: {similar_error}") + else: + logger.debug(f"Skipping similar artists for {watchlist_artist.artist_name} (iTunes doesn't support this feature)") return ScanResult( artist_name=watchlist_artist.artist_name, - spotify_artist_id=watchlist_artist.spotify_artist_id, + spotify_artist_id=watchlist_artist.spotify_artist_id or '', albums_checked=len(albums), new_tracks_found=new_tracks_found, tracks_added_to_wishlist=tracks_added_to_wishlist, @@ -534,7 +585,7 @@ class WatchlistScanner: logger.error(f"Error scanning artist {watchlist_artist.artist_name}: {e}") return ScanResult( artist_name=watchlist_artist.artist_name, - spotify_artist_id=watchlist_artist.spotify_artist_id, + spotify_artist_id=watchlist_artist.spotify_artist_id or '', albums_checked=0, new_tracks_found=0, tracks_added_to_wishlist=0, @@ -592,6 +643,57 @@ class WatchlistScanner: logger.error(f"Error getting discography for artist {spotify_artist_id}: {e}") return None + def _get_artist_discography_with_client(self, client, artist_id: str, last_scan_timestamp: Optional[datetime] = None) -> Optional[List]: + """ + Get artist's discography using the specified client, optionally filtered by release date. + + Args: + client: The metadata client to use (spotify or itunes) + artist_id: Artist ID for the given client + last_scan_timestamp: Only return releases after this date (for incremental scans) + If None, uses lookback period setting from database + """ + try: + # Get all artist albums (albums + singles) + logger.debug(f"Fetching discography for artist {artist_id}") + albums = client.get_artist_albums(artist_id, album_type='album,single', limit=50) + + if not albums: + logger.warning(f"No albums found for artist {artist_id}") + return [] + + # Add small delay after fetching artist discography to be extra safe + time.sleep(0.3) # 300ms breathing room + + # Determine cutoff date for filtering + cutoff_timestamp = last_scan_timestamp + + # If no last scan timestamp, use lookback period setting + if cutoff_timestamp is None: + lookback_period = self._get_lookback_period_setting() + if lookback_period != 'all': + # Convert period to days and create cutoff date (use UTC) + days = int(lookback_period) + cutoff_timestamp = datetime.now(timezone.utc) - timedelta(days=days) + logger.info(f"Using lookback period: {lookback_period} days (cutoff: {cutoff_timestamp})") + + # Filter by release date if we have a cutoff timestamp + if cutoff_timestamp: + filtered_albums = [] + for album in albums: + if self.is_album_after_timestamp(album, cutoff_timestamp): + filtered_albums.append(album) + + logger.info(f"Filtered {len(albums)} albums to {len(filtered_albums)} released after {cutoff_timestamp}") + return filtered_albums + + # Return all albums if no cutoff (lookback_period = 'all') + return albums + + except Exception as e: + logger.error(f"Error getting discography for artist {artist_id}: {e}") + return None + def _backfill_missing_ids(self, artists: List[WatchlistArtist], provider: str): """ Proactively match ALL artists missing IDs for the current provider. From f12478ee70a4c3e873f774cbe6adfbb2bb2447b2 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Thu, 22 Jan 2026 19:12:14 -0800 Subject: [PATCH 04/20] Add iTunes fallback and improve artist/album handling Adds iTunes fallback to SpotifyClient for search and metadata when Spotify is not authenticated. Updates album type logic to distinguish EPs, singles, and albums more accurately. Refactors watchlist database methods to support both Spotify and iTunes artist IDs. Improves deduplication and normalization of album names from iTunes. Updates web server and frontend to use new album type logic and support both ID types. Adds artist bubble snapshot example data. --- core/itunes_client.py | 98 +++++++--- core/spotify_client.py | 372 ++++++++++++++++++++----------------- database/music_database.py | 93 ++++++---- web_server.py | 5 +- webui/static/script.js | 6 +- 5 files changed, 349 insertions(+), 225 deletions(-) diff --git a/core/itunes_client.py b/core/itunes_client.py index 7f430d0a..e799fa74 100644 --- a/core/itunes_client.py +++ b/core/itunes_client.py @@ -141,13 +141,13 @@ class Album: # Determine album type from collection type track_count = album_data.get('trackCount', 0) - + # iTunes doesn't clearly distinguish EPs, but we can infer: # Singles typically have 1-3 tracks, EPs have 4-6, Albums have 7+ if track_count <= 3: album_type = 'single' elif track_count <= 6: - album_type = 'single' # iTunes calls EPs "albums" but we can mark shorter ones + album_type = 'ep' # 4-6 tracks = EP else: album_type = 'album' @@ -371,7 +371,7 @@ class iTunesClient: if track_count <= 3: album_type = 'single' elif track_count <= 6: - album_type = 'single' # EP treated as single + album_type = 'ep' # 4-6 tracks = EP else: album_type = 'album' @@ -433,17 +433,42 @@ class iTunesClient: # ==================== Artist Methods ==================== + def _get_artist_image_from_albums(self, artist_id: str) -> Optional[str]: + """ + Get artist image by fetching their first album's artwork. + iTunes doesn't reliably return artist images, so we use album art as fallback. + """ + try: + # Lookup is not rate-limited, so this is fast + results = self._lookup(id=artist_id, entity='album', limit=1) + + for item in results: + if item.get('wrapperType') == 'collection' and item.get('artworkUrl100'): + # Return high-res version + return item['artworkUrl100'].replace('100x100bb', '600x600bb') + except Exception as e: + logger.debug(f"Could not fetch album art for artist {artist_id}: {e}") + + return None + @rate_limited def search_artists(self, query: str, limit: int = 20) -> List[Artist]: - """Search for artists using iTunes API""" + """Search for artists using iTunes API - includes album art fallback for images""" results = self._search(query, 'musicArtist', limit) artists = [] - + for artist_data in results: if artist_data.get('wrapperType') == 'artist': artist = Artist.from_itunes_artist(artist_data) + + # If no artist image, try to get their first album's artwork + if not artist.image_url: + album_art = self._get_artist_image_from_albums(str(artist_data.get('artistId', ''))) + if album_art: + artist.image_url = album_art + artists.append(artist) - + return artists def get_artist(self, artist_id: str) -> Optional[Dict[str, Any]]: @@ -461,14 +486,22 @@ class iTunesClient: for artist_data in results: if artist_data.get('wrapperType') == 'artist': # Build images array - iTunes artist search doesn't reliably return images - # but we include the structure for compatibility + # Use album art as fallback images = [] - if artist_data.get('artworkUrl100'): - artwork_base = artist_data['artworkUrl100'] + artwork_url = artist_data.get('artworkUrl100') + + # If no artist artwork, try to get from their first album + if not artwork_url: + album_art = self._get_artist_image_from_albums(str(artist_data.get('artistId', ''))) + if album_art: + # Convert back to base URL format for building array + artwork_url = album_art.replace('600x600bb', '100x100bb') + + if artwork_url: images = [ - {'url': artwork_base.replace('100x100bb', '600x600bb'), 'height': 600, 'width': 600}, - {'url': artwork_base.replace('100x100bb', '300x300bb'), 'height': 300, 'width': 300}, - {'url': artwork_base, 'height': 100, 'width': 100} + {'url': artwork_url.replace('100x100bb', '600x600bb'), 'height': 600, 'width': 600}, + {'url': artwork_url.replace('100x100bb', '300x300bb'), 'height': 300, 'width': 300}, + {'url': artwork_url, 'height': 100, 'width': 100} ] # Get genre @@ -494,26 +527,49 @@ class iTunesClient: def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 50) -> List[Album]: """ Get albums by artist ID - + Note: iTunes doesn't support filtering by album_type in the same way as Spotify, so we fetch all albums and can filter client-side if needed. """ + import re + results = self._lookup(id=artist_id, entity='album', limit=min(limit, 200)) albums = [] - + seen_albums = set() # Track normalized names to prevent duplicates + + def normalize_album_name(name: str) -> str: + """Normalize album name for deduplication (removes edition suffixes, etc.)""" + normalized = name.lower().strip() + # Remove common edition suffixes + normalized = re.sub(r'\s*[\(\[]\s*(deluxe|explicit|clean|remaster|expanded|anniversary|edition|version|bonus|special|standard).*?[\)\]]', '', normalized, flags=re.IGNORECASE) + # Remove trailing edition keywords without brackets + normalized = re.sub(r'\s*[-–—]\s*(deluxe|explicit|clean|remaster|expanded|anniversary|edition|version).*$', '', normalized, flags=re.IGNORECASE) + # Normalize whitespace + normalized = re.sub(r'\s+', ' ', normalized).strip() + return normalized + for album_data in results: if album_data.get('wrapperType') == 'collection': album = Album.from_itunes_album(album_data) - - # Filter by album_type if specified + + # Filter by album_type if specified (now includes 'ep') if album_type != 'album,single': - requested_types = album_type.split(',') + requested_types = [t.strip() for t in album_type.split(',')] + # Also accept 'ep' when 'single' is requested (for backward compat) if album.album_type not in requested_types: - continue - + if not (album.album_type == 'ep' and 'single' in requested_types): + continue + + # Deduplicate by normalized name + normalized_name = normalize_album_name(album.name) + if normalized_name in seen_albums: + logger.debug(f"Skipping duplicate album: {album.name} (normalized: {normalized_name})") + continue + + seen_albums.add(normalized_name) albums.append(album) - - logger.info(f"Retrieved {len(albums)} albums for artist {artist_id}") + + logger.info(f"Retrieved {len(albums)} unique albums for artist {artist_id} (filtered from {len(results)} results)") return albums[:limit] # ==================== Playlist Methods ==================== diff --git a/core/spotify_client.py b/core/spotify_client.py index e709be6f..304f39de 100644 --- a/core/spotify_client.py +++ b/core/spotify_client.py @@ -169,8 +169,18 @@ class SpotifyClient: def __init__(self): self.sp: Optional[spotipy.Spotify] = None self.user_id: Optional[str] = None + self._itunes_client = None # Lazy-loaded iTunes fallback self._setup_client() + @property + def _itunes(self): + """Lazy-load iTunes client for fallback when Spotify not authenticated""" + if self._itunes_client is None: + from core.itunes_client import iTunesClient + self._itunes_client = iTunesClient() + logger.info("iTunes fallback client initialized") + return self._itunes_client + def reload_config(self): """Reload configuration and re-initialize client""" self._setup_client() @@ -201,10 +211,23 @@ class SpotifyClient: self.sp = None def is_authenticated(self) -> bool: - """Check if Spotify client is authenticated and working""" + """ + Check if client can service metadata requests. + Returns True if Spotify is authenticated OR iTunes fallback is available. + For Spotify-specific auth check, use is_spotify_authenticated(). + """ + # If Spotify is authenticated, we're good + if self.is_spotify_authenticated(): + return True + + # iTunes fallback is always available + return True + + def is_spotify_authenticated(self) -> bool: + """Check if Spotify client is specifically authenticated (not just iTunes fallback)""" if self.sp is None: return False - + try: # Make a simple API call to verify authentication self.sp.current_user() @@ -228,7 +251,7 @@ class SpotifyClient: @rate_limited def get_user_playlists(self) -> List[Playlist]: - if not self.is_authenticated(): + if not self.is_spotify_authenticated(): logger.error("Not authenticated with Spotify") return [] @@ -262,7 +285,7 @@ class SpotifyClient: @rate_limited def get_user_playlists_metadata_only(self) -> List[Playlist]: """Get playlists without fetching all track details for faster loading""" - if not self.is_authenticated(): + if not self.is_spotify_authenticated(): logger.error("Not authenticated with Spotify") return [] @@ -312,7 +335,7 @@ class SpotifyClient: @rate_limited def get_saved_tracks_count(self) -> int: """Get the total count of user's saved/liked songs without fetching all tracks""" - if not self.is_authenticated(): + if not self.is_spotify_authenticated(): logger.error("Not authenticated with Spotify") return 0 @@ -331,7 +354,7 @@ class SpotifyClient: @rate_limited def get_saved_tracks(self) -> List[Track]: """Fetch all user's saved/liked songs from Spotify""" - if not self.is_authenticated(): + if not self.is_spotify_authenticated(): logger.error("Not authenticated with Spotify") return [] @@ -373,7 +396,7 @@ class SpotifyClient: @rate_limited def _get_playlist_tracks(self, playlist_id: str) -> List[Track]: - if not self.is_authenticated(): + if not self.is_spotify_authenticated(): return [] tracks = [] @@ -397,7 +420,7 @@ class SpotifyClient: @rate_limited def get_playlist_by_id(self, playlist_id: str) -> Optional[Playlist]: - if not self.is_authenticated(): + if not self.is_spotify_authenticated(): return None try: @@ -411,104 +434,113 @@ class SpotifyClient: @rate_limited def search_tracks(self, query: str, limit: int = 20) -> List[Track]: - if not self.is_authenticated(): - return [] - - try: - results = self.sp.search(q=query, type='track', limit=limit) - tracks = [] - - for track_data in results['tracks']['items']: - track = Track.from_spotify_track(track_data) - tracks.append(track) - - return tracks - - except Exception as e: - logger.error(f"Error searching tracks: {e}") - return [] - + """Search for tracks - falls back to iTunes if Spotify not authenticated""" + if self.is_spotify_authenticated(): + try: + results = self.sp.search(q=query, type='track', limit=limit) + tracks = [] + + for track_data in results['tracks']['items']: + track = Track.from_spotify_track(track_data) + tracks.append(track) + + return tracks + + except Exception as e: + logger.error(f"Error searching tracks via Spotify: {e}") + # Fall through to iTunes fallback + + # iTunes fallback + logger.debug(f"Using iTunes fallback for track search: {query}") + return self._itunes.search_tracks(query, limit) + @rate_limited def search_artists(self, query: str, limit: int = 20) -> List[Artist]: - """Search for artists using Spotify API""" - if not self.is_authenticated(): - return [] - - try: - results = self.sp.search(q=query, type='artist', limit=limit) - artists = [] - - for artist_data in results['artists']['items']: - artist = Artist.from_spotify_artist(artist_data) - artists.append(artist) - - return artists - - except Exception as e: - logger.error(f"Error searching artists: {e}") - return [] - + """Search for artists - falls back to iTunes if Spotify not authenticated""" + if self.is_spotify_authenticated(): + try: + results = self.sp.search(q=query, type='artist', limit=limit) + artists = [] + + for artist_data in results['artists']['items']: + artist = Artist.from_spotify_artist(artist_data) + artists.append(artist) + + return artists + + except Exception as e: + logger.error(f"Error searching artists via Spotify: {e}") + # Fall through to iTunes fallback + + # iTunes fallback + logger.debug(f"Using iTunes fallback for artist search: {query}") + return self._itunes.search_artists(query, limit) + @rate_limited def search_albums(self, query: str, limit: int = 20) -> List[Album]: - """Search for albums using Spotify API""" - if not self.is_authenticated(): - return [] - - try: - results = self.sp.search(q=query, type='album', limit=limit) - albums = [] - - for album_data in results['albums']['items']: - album = Album.from_spotify_album(album_data) - albums.append(album) - - return albums - - except Exception as e: - logger.error(f"Error searching albums: {e}") - return [] + """Search for albums - falls back to iTunes if Spotify not authenticated""" + if self.is_spotify_authenticated(): + try: + results = self.sp.search(q=query, type='album', limit=limit) + albums = [] + + for album_data in results['albums']['items']: + album = Album.from_spotify_album(album_data) + albums.append(album) + + return albums + + except Exception as e: + logger.error(f"Error searching albums via Spotify: {e}") + # Fall through to iTunes fallback + + # iTunes fallback + logger.debug(f"Using iTunes fallback for album search: {query}") + return self._itunes.search_albums(query, limit) @rate_limited def get_track_details(self, track_id: str) -> Optional[Dict[str, Any]]: - """Get detailed track information including album data and track number""" - if not self.is_authenticated(): - return None - - try: - track_data = self.sp.track(track_id) - - # Enhance with additional useful metadata for our purposes - if track_data: - enhanced_data = { - 'id': track_data['id'], - 'name': track_data['name'], - 'track_number': track_data['track_number'], - 'disc_number': track_data['disc_number'], - 'duration_ms': track_data['duration_ms'], - 'explicit': track_data['explicit'], - 'artists': [artist['name'] for artist in track_data['artists']], - 'primary_artist': track_data['artists'][0]['name'] if track_data['artists'] else None, - 'album': { - 'id': track_data['album']['id'], - 'name': track_data['album']['name'], - 'total_tracks': track_data['album']['total_tracks'], - 'release_date': track_data['album']['release_date'], - 'album_type': track_data['album']['album_type'], - 'artists': [artist['name'] for artist in track_data['album']['artists']] - }, - 'is_album_track': track_data['album']['total_tracks'] > 1, - 'raw_data': track_data # Keep original for fallback - } - return enhanced_data - return track_data - - except Exception as e: - logger.error(f"Error fetching track details: {e}") - return None + """Get detailed track information - falls back to iTunes if Spotify not authenticated""" + if self.is_spotify_authenticated(): + try: + track_data = self.sp.track(track_id) + + # Enhance with additional useful metadata for our purposes + if track_data: + enhanced_data = { + 'id': track_data['id'], + 'name': track_data['name'], + 'track_number': track_data['track_number'], + 'disc_number': track_data['disc_number'], + 'duration_ms': track_data['duration_ms'], + 'explicit': track_data['explicit'], + 'artists': [artist['name'] for artist in track_data['artists']], + 'primary_artist': track_data['artists'][0]['name'] if track_data['artists'] else None, + 'album': { + 'id': track_data['album']['id'], + 'name': track_data['album']['name'], + 'total_tracks': track_data['album']['total_tracks'], + 'release_date': track_data['album']['release_date'], + 'album_type': track_data['album']['album_type'], + 'artists': [artist['name'] for artist in track_data['album']['artists']] + }, + 'is_album_track': track_data['album']['total_tracks'] > 1, + 'raw_data': track_data # Keep original for fallback + } + return enhanced_data + return track_data + + except Exception as e: + logger.error(f"Error fetching track details via Spotify: {e}") + # Fall through to iTunes fallback + + # iTunes fallback + logger.debug(f"Using iTunes fallback for track details: {track_id}") + return self._itunes.get_track_details(track_id) @rate_limited def get_track_features(self, track_id: str) -> Optional[Dict[str, Any]]: - if not self.is_authenticated(): + if not self.is_spotify_authenticated(): return None try: @@ -521,83 +553,89 @@ class SpotifyClient: @rate_limited def get_album(self, album_id: str) -> Optional[Dict[str, Any]]: - """Get album information including tracks""" - if not self.is_authenticated(): - return None - - try: - album_data = self.sp.album(album_id) - return album_data - - except Exception as e: - logger.error(f"Error fetching album: {e}") - return None + """Get album information - falls back to iTunes if Spotify not authenticated""" + if self.is_spotify_authenticated(): + try: + album_data = self.sp.album(album_id) + return album_data + + except Exception as e: + logger.error(f"Error fetching album via Spotify: {e}") + # Fall through to iTunes fallback + + # iTunes fallback + logger.debug(f"Using iTunes fallback for album: {album_id}") + return self._itunes.get_album(album_id) @rate_limited def get_album_tracks(self, album_id: str) -> Optional[Dict[str, Any]]: - """Get album tracks with pagination to fetch all tracks""" - if not self.is_authenticated(): - return None + """Get album tracks - falls back to iTunes if Spotify not authenticated""" + if self.is_spotify_authenticated(): + try: + # Get first page of tracks + first_page = self.sp.album_tracks(album_id) + if not first_page or 'items' not in first_page: + return None - try: - # Get first page of tracks - first_page = self.sp.album_tracks(album_id) - if not first_page or 'items' not in first_page: - return None + # Collect all tracks starting with first page + all_tracks = first_page['items'][:] - # Collect all tracks starting with first page - all_tracks = first_page['items'][:] + # Fetch remaining pages if they exist + next_page = first_page + while next_page.get('next'): + next_page = self.sp.next(next_page) + if next_page and 'items' in next_page: + all_tracks.extend(next_page['items']) - # Fetch remaining pages if they exist - next_page = first_page - while next_page.get('next'): - next_page = self.sp.next(next_page) - if next_page and 'items' in next_page: - all_tracks.extend(next_page['items']) + # Log success + logger.info(f"Retrieved {len(all_tracks)} tracks for album {album_id}") - # Log success - logger.info(f"Retrieved {len(all_tracks)} tracks for album {album_id}") + # Return structure with all tracks + result = first_page.copy() + result['items'] = all_tracks + result['next'] = None # No more pages + result['limit'] = len(all_tracks) # Update to reflect all tracks fetched - # Return structure with all tracks - result = first_page.copy() - result['items'] = all_tracks - result['next'] = None # No more pages - result['limit'] = len(all_tracks) # Update to reflect all tracks fetched + return result - return result + except Exception as e: + logger.error(f"Error fetching album tracks via Spotify: {e}") + # Fall through to iTunes fallback - except Exception as e: - logger.error(f"Error fetching album tracks: {e}") - return None + # iTunes fallback + logger.debug(f"Using iTunes fallback for album tracks: {album_id}") + return self._itunes.get_album_tracks(album_id) @rate_limited def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 50) -> List[Album]: - """Get albums by artist ID""" - if not self.is_authenticated(): - return [] - - try: - albums = [] - results = self.sp.artist_albums(artist_id, album_type=album_type, limit=limit) - - while results: - for album_data in results['items']: - album = Album.from_spotify_album(album_data) - albums.append(album) - - # Get next batch if available - results = self.sp.next(results) if results['next'] else None - - logger.info(f"Retrieved {len(albums)} albums for artist {artist_id}") - return albums - - except Exception as e: - logger.error(f"Error fetching artist albums: {e}") - return [] + """Get albums by artist ID - falls back to iTunes if Spotify not authenticated""" + if self.is_spotify_authenticated(): + try: + albums = [] + results = self.sp.artist_albums(artist_id, album_type=album_type, limit=limit) + + while results: + for album_data in results['items']: + album = Album.from_spotify_album(album_data) + albums.append(album) + + # Get next batch if available + results = self.sp.next(results) if results['next'] else None + + logger.info(f"Retrieved {len(albums)} albums for artist {artist_id}") + return albums + + except Exception as e: + logger.error(f"Error fetching artist albums via Spotify: {e}") + # Fall through to iTunes fallback + + # iTunes fallback + logger.debug(f"Using iTunes fallback for artist albums: {artist_id}") + return self._itunes.get_artist_albums(artist_id, album_type, limit) @rate_limited def get_user_info(self) -> Optional[Dict[str, Any]]: - if not self.is_authenticated(): + if not self.is_spotify_authenticated(): return None try: @@ -609,19 +647,21 @@ class SpotifyClient: @rate_limited def get_artist(self, artist_id: str) -> Optional[Dict[str, Any]]: """ - Get full artist details from Spotify API. + Get full artist details - falls back to iTunes if Spotify not authenticated. Args: - artist_id: Spotify artist ID + artist_id: Artist ID (Spotify or iTunes depending on authentication) Returns: Dictionary with artist data including images, genres, popularity """ - if not self.is_authenticated(): - return None + if self.is_spotify_authenticated(): + try: + return self.sp.artist(artist_id) + except Exception as e: + logger.error(f"Error fetching artist via Spotify: {e}") + # Fall through to iTunes fallback - try: - return self.sp.artist(artist_id) - except Exception as e: - logger.error(f"Error fetching artist {artist_id}: {e}") - return None \ No newline at end of file + # iTunes fallback + logger.debug(f"Using iTunes fallback for artist: {artist_id}") + return self._itunes.get_artist(artist_id) \ No newline at end of file diff --git a/database/music_database.py b/database/music_database.py index 71f612cb..2f774d97 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -2700,64 +2700,89 @@ class MusicDatabase: return 0 # Watchlist operations - def add_artist_to_watchlist(self, spotify_artist_id: str, artist_name: str) -> bool: - """Add an artist to the watchlist for monitoring new releases""" + def add_artist_to_watchlist(self, artist_id: str, artist_name: str) -> bool: + """Add an artist to the watchlist for monitoring new releases. + + Automatically detects if artist_id is a Spotify ID (alphanumeric) or iTunes ID (numeric). + """ try: with self._get_connection() as conn: cursor = conn.cursor() - - cursor.execute(""" - INSERT OR REPLACE INTO watchlist_artists - (spotify_artist_id, artist_name, date_added, updated_at) - VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - """, (spotify_artist_id, artist_name)) - + + # Detect ID type: iTunes IDs are purely numeric, Spotify IDs are alphanumeric + is_itunes_id = artist_id.isdigit() + + if is_itunes_id: + cursor.execute(""" + INSERT OR REPLACE INTO watchlist_artists + (itunes_artist_id, artist_name, date_added, updated_at) + VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, (artist_id, artist_name)) + logger.info(f"Added artist '{artist_name}' to watchlist (iTunes ID: {artist_id})") + else: + cursor.execute(""" + INSERT OR REPLACE INTO watchlist_artists + (spotify_artist_id, artist_name, date_added, updated_at) + VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, (artist_id, artist_name)) + logger.info(f"Added artist '{artist_name}' to watchlist (Spotify ID: {artist_id})") + conn.commit() - logger.info(f"Added artist '{artist_name}' to watchlist (Spotify ID: {spotify_artist_id})") return True - + except Exception as e: logger.error(f"Error adding artist '{artist_name}' to watchlist: {e}") return False - def remove_artist_from_watchlist(self, spotify_artist_id: str) -> bool: - """Remove an artist from the watchlist""" + def remove_artist_from_watchlist(self, artist_id: str) -> bool: + """Remove an artist from the watchlist (checks both Spotify and iTunes IDs)""" try: with self._get_connection() as conn: cursor = conn.cursor() - - # Get artist name for logging - cursor.execute("SELECT artist_name FROM watchlist_artists WHERE spotify_artist_id = ?", (spotify_artist_id,)) + + # Get artist name for logging (check both ID columns) + cursor.execute(""" + SELECT artist_name FROM watchlist_artists + WHERE spotify_artist_id = ? OR itunes_artist_id = ? + """, (artist_id, artist_id)) result = cursor.fetchone() artist_name = result['artist_name'] if result else "Unknown" - - cursor.execute("DELETE FROM watchlist_artists WHERE spotify_artist_id = ?", (spotify_artist_id,)) - + + cursor.execute(""" + DELETE FROM watchlist_artists + WHERE spotify_artist_id = ? OR itunes_artist_id = ? + """, (artist_id, artist_id)) + if cursor.rowcount > 0: conn.commit() - logger.info(f"Removed artist '{artist_name}' from watchlist (Spotify ID: {spotify_artist_id})") + logger.info(f"Removed artist '{artist_name}' from watchlist (ID: {artist_id})") return True else: - logger.warning(f"Artist with Spotify ID {spotify_artist_id} not found in watchlist") + logger.warning(f"Artist with ID {artist_id} not found in watchlist") return False - + except Exception as e: - logger.error(f"Error removing artist from watchlist (Spotify ID: {spotify_artist_id}): {e}") + logger.error(f"Error removing artist from watchlist (ID: {artist_id}): {e}") return False - def is_artist_in_watchlist(self, spotify_artist_id: str) -> bool: - """Check if an artist is currently in the watchlist""" + def is_artist_in_watchlist(self, artist_id: str) -> bool: + """Check if an artist is currently in the watchlist (checks both Spotify and iTunes IDs)""" try: with self._get_connection() as conn: cursor = conn.cursor() - - cursor.execute("SELECT 1 FROM watchlist_artists WHERE spotify_artist_id = ? LIMIT 1", (spotify_artist_id,)) + + # Check both spotify_artist_id and itunes_artist_id columns + cursor.execute(""" + SELECT 1 FROM watchlist_artists + WHERE spotify_artist_id = ? OR itunes_artist_id = ? + LIMIT 1 + """, (artist_id, artist_id)) result = cursor.fetchone() - + return result is not None - + except Exception as e: - logger.error(f"Error checking if artist is in watchlist (Spotify ID: {spotify_artist_id}): {e}") + logger.error(f"Error checking if artist is in watchlist (ID: {artist_id}): {e}") return False def get_watchlist_artists(self) -> List[WatchlistArtist]: @@ -2839,8 +2864,8 @@ class MusicDatabase: logger.error(f"Error getting watchlist count: {e}") return 0 - def update_watchlist_artist_image(self, spotify_artist_id: str, image_url: str) -> bool: - """Update the image URL for a watchlist artist""" + def update_watchlist_artist_image(self, artist_id: str, image_url: str) -> bool: + """Update the image URL for a watchlist artist (checks both Spotify and iTunes IDs)""" try: with self._get_connection() as conn: cursor = conn.cursor() @@ -2856,8 +2881,8 @@ class MusicDatabase: cursor.execute(""" UPDATE watchlist_artists SET image_url = ?, updated_at = CURRENT_TIMESTAMP - WHERE spotify_artist_id = ? - """, (image_url, spotify_artist_id)) + WHERE spotify_artist_id = ? OR itunes_artist_id = ? + """, (image_url, artist_id, artist_id)) conn.commit() return cursor.rowcount > 0 diff --git a/web_server.py b/web_server.py index d83bb7b4..93b341bb 100644 --- a/web_server.py +++ b/web_server.py @@ -22684,8 +22684,11 @@ def get_spotify_artist_discography(artist_name): if album_type == 'single' or track_count <= 3: singles.append(release_data) - elif album_type == 'compilation' or (track_count <= 8 and track_count > 3): + elif album_type == 'ep' or (track_count >= 4 and track_count <= 6): eps.append(release_data) + elif album_type == 'compilation': + # Compilations go with albums + albums.append(release_data) else: albums.append(release_data) diff --git a/webui/static/script.js b/webui/static/script.js index 1ff3dc19..5cff6c6e 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -25411,7 +25411,7 @@ function createReleaseCard(release) { name: release.title, image_url: release.image_url, release_date: release.year ? `${release.year}-01-01` : '', - album_type: release.type || 'album', + album_type: release.album_type || release.type || 'album', total_tracks: (release.track_completion && typeof release.track_completion === 'object') ? release.track_completion.total_tracks : 1 }; @@ -25440,8 +25440,8 @@ function createReleaseCard(release) { throw new Error('No tracks found for this release'); } - // Determine album type based on release data - const albumType = release.type === 'single' ? 'singles' : 'albums'; + // Use the actual album type from release data + const albumType = release.album_type || release.type || 'album'; // Open the Add to Wishlist modal // Note: openAddToWishlistModal has its own loading overlay From fecd371b5ec21fbbafca52d13aa93a85cc0bbea9 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Fri, 23 Jan 2026 07:38:03 -0800 Subject: [PATCH 05/20] Add lazy loading for artist images across UI Implements lazy loading of artist images in search results, artist pages, and similar artist bubbles to improve performance and user experience. Updates the iTunes client to prefer explicit album versions and deduplicate albums accordingly. Adds a new API endpoint to fetch artist images, and updates frontend logic to asynchronously fetch and display images where missing. --- core/itunes_client.py | 103 ++++++++++++------- web_server.py | 36 ++++++- webui/static/script.js | 218 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 317 insertions(+), 40 deletions(-) diff --git a/core/itunes_client.py b/core/itunes_client.py index e799fa74..cf974687 100644 --- a/core/itunes_client.py +++ b/core/itunes_client.py @@ -228,7 +228,8 @@ class iTunesClient: 'country': self.country, 'media': 'music', 'entity': entity, - 'limit': min(limit, 200) # iTunes max is 200 + 'limit': min(limit, 200), # iTunes max is 200 + 'explicit': 'Yes' # Include explicit content (prefer over clean versions) } response = self.session.get( @@ -335,16 +336,40 @@ class iTunesClient: @rate_limited def search_albums(self, query: str, limit: int = 20) -> List[Album]: - """Search for albums using iTunes API""" - results = self._search(query, 'album', limit) + """Search for albums using iTunes API. + + Filters out clean versions when explicit versions are available. + """ + results = self._search(query, 'album', limit * 2) # Fetch more to account for filtering albums = [] - + seen_albums = {} # Track albums by normalized name to prefer explicit versions + for album_data in results: - if album_data.get('wrapperType') == 'collection': - album = Album.from_itunes_album(album_data) - albums.append(album) - - return albums + if album_data.get('wrapperType') != 'collection': + continue + + # Get album name and explicitness + album_name = album_data.get('collectionName', '').lower().strip() + artist_name = album_data.get('artistName', '').lower().strip() + is_explicit = album_data.get('collectionExplicitness') == 'explicit' + + # Create a key for deduplication (album name + artist) + key = f"{album_name}|{artist_name}" + + # If we've seen this album before + if key in seen_albums: + # Only replace if current one is explicit and previous was clean + if is_explicit and not seen_albums[key]['is_explicit']: + seen_albums[key] = {'data': album_data, 'is_explicit': is_explicit} + else: + seen_albums[key] = {'data': album_data, 'is_explicit': is_explicit} + + # Convert to Album objects + for item in seen_albums.values(): + album = Album.from_itunes_album(item['data']) + albums.append(album) + + return albums[:limit] def get_album(self, album_id: str) -> Optional[Dict[str, Any]]: """Get album information - normalized to Spotify format""" @@ -453,20 +478,17 @@ class iTunesClient: @rate_limited def search_artists(self, query: str, limit: int = 20) -> List[Artist]: - """Search for artists using iTunes API - includes album art fallback for images""" + """Search for artists using iTunes API. + + Note: Artist images are not fetched during search to keep it fast. + Images are fetched when viewing artist details (get_artist method). + """ results = self._search(query, 'musicArtist', limit) artists = [] for artist_data in results: if artist_data.get('wrapperType') == 'artist': artist = Artist.from_itunes_artist(artist_data) - - # If no artist image, try to get their first album's artwork - if not artist.image_url: - album_art = self._get_artist_image_from_albums(str(artist_data.get('artistId', ''))) - if album_art: - artist.image_url = album_art - artists.append(artist) return artists @@ -530,12 +552,12 @@ class iTunesClient: Note: iTunes doesn't support filtering by album_type in the same way as Spotify, so we fetch all albums and can filter client-side if needed. + Prefers explicit versions over clean versions when both exist. """ import re results = self._lookup(id=artist_id, entity='album', limit=min(limit, 200)) - albums = [] - seen_albums = set() # Track normalized names to prevent duplicates + seen_albums = {} # Track albums by normalized name, prefer explicit versions def normalize_album_name(name: str) -> str: """Normalize album name for deduplication (removes edition suffixes, etc.)""" @@ -549,25 +571,38 @@ class iTunesClient: return normalized for album_data in results: - if album_data.get('wrapperType') == 'collection': - album = Album.from_itunes_album(album_data) + if album_data.get('wrapperType') != 'collection': + continue - # Filter by album_type if specified (now includes 'ep') - if album_type != 'album,single': - requested_types = [t.strip() for t in album_type.split(',')] - # Also accept 'ep' when 'single' is requested (for backward compat) - if album.album_type not in requested_types: - if not (album.album_type == 'ep' and 'single' in requested_types): - continue + # Check if explicit + is_explicit = album_data.get('collectionExplicitness') == 'explicit' - # Deduplicate by normalized name - normalized_name = normalize_album_name(album.name) - if normalized_name in seen_albums: + # Create album object + album = Album.from_itunes_album(album_data) + + # Filter by album_type if specified (now includes 'ep') + if album_type != 'album,single': + requested_types = [t.strip() for t in album_type.split(',')] + # Also accept 'ep' when 'single' is requested (for backward compat) + if album.album_type not in requested_types: + if not (album.album_type == 'ep' and 'single' in requested_types): + continue + + # Deduplicate by normalized name, prefer explicit versions + normalized_name = normalize_album_name(album.name) + + if normalized_name in seen_albums: + # Only replace if current one is explicit and previous was clean + if is_explicit and not seen_albums[normalized_name]['is_explicit']: + logger.debug(f"Replacing clean version with explicit: {album.name}") + seen_albums[normalized_name] = {'album': album, 'is_explicit': is_explicit} + else: logger.debug(f"Skipping duplicate album: {album.name} (normalized: {normalized_name})") - continue + else: + seen_albums[normalized_name] = {'album': album, 'is_explicit': is_explicit} - seen_albums.add(normalized_name) - albums.append(album) + # Extract albums from dict + albums = [item['album'] for item in seen_albums.values()] logger.info(f"Retrieved {len(albums)} unique albums for artist {artist_id} (filtered from {len(results)} results)") return albums[:limit] diff --git a/web_server.py b/web_server.py index 93b341bb..753a598d 100644 --- a/web_server.py +++ b/web_server.py @@ -5017,6 +5017,31 @@ def get_similar_artists(artist_name): "error": str(e) }), 500 +@app.route('/api/artist//image', methods=['GET']) +def get_artist_image(artist_id): + """Get artist image URL - used for lazy loading in search results. + + For iTunes, this fetches the artist's first album artwork as a fallback. + For Spotify, returns the artist's image directly. + """ + try: + if spotify_client and spotify_client.is_spotify_authenticated(): + # Use Spotify directly + artist_data = spotify_client.sp.artist(artist_id) + if artist_data and artist_data.get('images'): + image_url = artist_data['images'][0]['url'] if artist_data['images'] else None + return jsonify({"success": True, "image_url": image_url}) + return jsonify({"success": True, "image_url": None}) + else: + # Use iTunes fallback - fetch album art + from core.itunes_client import iTunesClient + itunes = iTunesClient() + image_url = itunes._get_artist_image_from_albums(artist_id) + return jsonify({"success": True, "image_url": image_url}) + except Exception as e: + print(f"Error fetching artist image: {e}") + return jsonify({"success": False, "image_url": None, "error": str(e)}) + @app.route('/api/artist//discography', methods=['GET']) def get_artist_discography(artist_id): """Get an artist's complete discography (albums and singles)""" @@ -14111,9 +14136,15 @@ def get_album_tracks(album_id): if not album_data: return jsonify({"error": "Album not found"}), 404 - # Extract tracks from album data + # Extract tracks from album data (Spotify format) tracks = album_data.get('tracks', {}).get('items', []) + # If no tracks in album data (iTunes format), fetch them separately + if not tracks: + tracks_data = spotify_client.get_album_tracks(album_id) + if tracks_data and 'items' in tracks_data: + tracks = tracks_data['items'] + # Format response album_dict = { 'id': album_data['id'], @@ -14121,12 +14152,13 @@ def get_album_tracks(album_id): 'artists': album_data.get('artists', []), 'release_date': album_data.get('release_date', ''), 'total_tracks': album_data.get('total_tracks', 0), - 'album_type': album_data.get('album_type', 'album'), # CRITICAL FIX: Include album_type for correct classification + 'album_type': album_data.get('album_type', 'album'), 'images': album_data.get('images', []), 'tracks': tracks } return jsonify(album_dict) except Exception as e: + logger.error(f"Error fetching album tracks: {e}") return jsonify({"error": str(e)}), 500 diff --git a/webui/static/script.js b/webui/static/script.js index 5cff6c6e..5aa92be5 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -2681,6 +2681,57 @@ function initializeSearchModeToggle() { }; } ); + + // Lazy load artist images that are missing + lazyLoadEnhancedSearchArtistImages(); + } + + // Lazy load artist images for enhanced search results + async function lazyLoadEnhancedSearchArtistImages() { + const artistLists = [ + document.getElementById('enh-db-artists-list'), + document.getElementById('enh-spotify-artists-list') + ]; + + for (const list of artistLists) { + if (!list) continue; + + const cardsNeedingImages = list.querySelectorAll('[data-needs-image="true"]'); + if (cardsNeedingImages.length === 0) continue; + + console.log(`πŸ–ΌοΈ Lazy loading ${cardsNeedingImages.length} artist images in enhanced search`); + + for (const card of cardsNeedingImages) { + const artistId = card.dataset.artistId; + if (!artistId) continue; + + try { + const response = await fetch(`/api/artist/${artistId}/image`); + const data = await response.json(); + + if (data.success && data.image_url) { + // Find the placeholder and replace with image + const placeholder = card.querySelector('.enh-item-image-placeholder'); + if (placeholder) { + const img = document.createElement('img'); + img.src = data.image_url; + img.className = 'enh-item-image artist-image'; + img.alt = card.querySelector('.enh-item-name')?.textContent || 'Artist'; + placeholder.replaceWith(img); + + // Apply dynamic glow + extractImageColors(data.image_url, (colors) => { + applyDynamicGlow(card, colors); + }); + } + card.dataset.needsImage = 'false'; + console.log(`βœ… Loaded image for artist ${artistId}`); + } + } catch (error) { + console.warn(`⚠️ Failed to load image for artist ${artistId}:`, error); + } + } + } } function formatDuration(durationMs) { @@ -2729,6 +2780,11 @@ function initializeSearchModeToggle() { // Add appropriate card class if (isArtist) { elem.className = 'enh-compact-item artist-card'; + // Add data attributes for lazy loading + if (item.id) { + elem.dataset.artistId = item.id; + elem.dataset.needsImage = config.image ? 'false' : 'true'; + } } else if (isAlbum) { elem.className = 'enh-compact-item album-card'; } else if (isTrack) { @@ -2752,7 +2808,7 @@ function initializeSearchModeToggle() { const imageHtml = config.image ? `${escapeHtml(config.name)}` - : `
${config.placeholder}
`; + : `
${config.placeholder}
`; const badgeHtml = config.badge ? `
${config.badge.text}
` @@ -11451,6 +11507,10 @@ function createArtistCard(artist, confidence) { const imageUrl = artist.image_url || ''; const confidencePercent = Math.round(confidence * 100); + // Add data attribute for lazy loading + card.dataset.artistId = artist.id; + card.dataset.needsImage = imageUrl ? 'false' : 'true'; + card.innerHTML = `
@@ -11683,6 +11743,16 @@ function renderArtistSearchResults(results) { console.error(`Error calling createArtistCard for result ${index}:`, error); } }); + + // Lazy load missing artist images + console.log('πŸ–ΌοΈ Starting lazy load for artist images in matching modal...'); + if (typeof lazyLoadArtistImages === 'function') { + lazyLoadArtistImages(container); + } else if (typeof window.lazyLoadArtistImages === 'function') { + window.lazyLoadArtistImages(container); + } else { + console.error('❌ lazyLoadArtistImages function not found!'); + } } function renderAlbumSearchResults(results) { @@ -19768,6 +19838,16 @@ function displayArtistsResults(query, results) { // Update watchlist status for all cards updateArtistCardWatchlistStatus(); + // Lazy load missing artist images + console.log('πŸ–ΌοΈ Starting lazy load for artist images on Artists page...'); + if (typeof lazyLoadArtistImages === 'function') { + lazyLoadArtistImages(container); + } else if (typeof window.lazyLoadArtistImages === 'function') { + window.lazyLoadArtistImages(container); + } else { + console.error('❌ lazyLoadArtistImages function not found!'); + } + // Add mouse wheel horizontal scrolling container.addEventListener('wheel', (event) => { if (event.deltaY !== 0) { @@ -19777,6 +19857,77 @@ function displayArtistsResults(query, results) { }); } +/** + * Lazy load artist images for cards that don't have images yet. + * Fetches images asynchronously so search results appear immediately. + */ +async function lazyLoadArtistImages(container) { + if (!container) { + console.error('❌ lazyLoadArtistImages: container is null'); + return; + } + + // Find all cards that need images + const cardsNeedingImages = container.querySelectorAll('[data-needs-image="true"]'); + + if (cardsNeedingImages.length === 0) { + console.log('βœ… All artist cards have images'); + return; + } + + console.log(`πŸ–ΌοΈ Lazy loading images for ${cardsNeedingImages.length} artist cards`); + + // Load images in parallel (but with a small batch to avoid overwhelming the server) + const batchSize = 5; + const cards = Array.from(cardsNeedingImages); + + for (let i = 0; i < cards.length; i += batchSize) { + const batch = cards.slice(i, i + batchSize); + + await Promise.all(batch.map(async (card) => { + const artistId = card.dataset.artistId; + if (!artistId) { + console.warn('⚠️ Card missing artistId:', card); + return; + } + + try { + console.log(`πŸ”„ Fetching image for artist ${artistId}...`); + const response = await fetch(`/api/artist/${artistId}/image`); + const data = await response.json(); + + console.log(`πŸ“₯ Got response for ${artistId}:`, data); + + if (data.success && data.image_url) { + // Update the card's background image + // Handle both card types (suggestion-card and artist-card) + if (card.classList.contains('suggestion-card')) { + card.style.backgroundImage = `url(${data.image_url})`; + card.style.backgroundSize = 'cover'; + card.style.backgroundPosition = 'center'; + } else if (card.classList.contains('artist-card')) { + const bgElement = card.querySelector('.artist-card-background'); + if (bgElement) { + // Clear the gradient first, then set the image + bgElement.style.cssText = `background-image: url('${data.image_url}'); background-size: cover; background-position: center;`; + } + } + + card.dataset.needsImage = 'false'; + console.log(`βœ… Loaded image for artist ${artistId}`); + } + } catch (error) { + console.error(`❌ Failed to load image for artist ${artistId}:`, error); + } + })); + } + + console.log('βœ… Finished lazy loading artist images'); +} + +// Make function globally accessible +window.lazyLoadArtistImages = lazyLoadArtistImages; + /** * Create HTML for an artist card */ @@ -19794,8 +19945,11 @@ function createArtistCardHTML(artist) { // Format popularity as a percentage for better UX const popularityText = popularity > 0 ? `${popularity}% Popular` : 'Popularity Unknown'; + // Track if image needs to be lazy loaded + const needsImage = imageUrl ? 'false' : 'true'; + return ` -
+
@@ -20107,6 +20261,9 @@ async function loadSimilarArtists(artistName) {
No similar artists found
`; + } else { + // Lazy load images for similar artists that don't have them + lazyLoadSimilarArtistImages(container); } } } catch (parseError) { @@ -20145,6 +20302,54 @@ async function loadSimilarArtists(artistName) { } } +/** + * Lazy load images for similar artist bubbles that don't have images + */ +async function lazyLoadSimilarArtistImages(container) { + if (!container) return; + + const bubblesNeedingImages = container.querySelectorAll('.similar-artist-bubble[data-needs-image="true"]'); + + if (bubblesNeedingImages.length === 0) { + console.log('βœ… All similar artist bubbles have images'); + return; + } + + console.log(`πŸ–ΌοΈ Lazy loading images for ${bubblesNeedingImages.length} similar artists`); + + // Load images in parallel batches + const batchSize = 5; + const bubbles = Array.from(bubblesNeedingImages); + + for (let i = 0; i < bubbles.length; i += batchSize) { + const batch = bubbles.slice(i, i + batchSize); + + await Promise.all(batch.map(async (bubble) => { + const artistId = bubble.getAttribute('data-artist-id'); + if (!artistId) return; + + try { + const response = await fetch(`/api/artist/${artistId}/image`); + const data = await response.json(); + + if (data.success && data.image_url) { + const imageContainer = bubble.querySelector('.similar-artist-bubble-image'); + if (imageContainer) { + const artistName = bubble.querySelector('.similar-artist-bubble-name')?.textContent || 'Artist'; + imageContainer.innerHTML = `${artistName}`; + bubble.setAttribute('data-needs-image', 'false'); + console.log(`βœ… Loaded image for similar artist ${artistId}`); + } + } + } catch (error) { + console.warn(`⚠️ Failed to load image for similar artist ${artistId}:`, error); + } + })); + } + + console.log('βœ… Finished lazy loading similar artist images'); +} + /** * Display similar artist bubble cards progressively (one at a time with delay) */ @@ -20206,11 +20411,15 @@ function createSimilarArtistBubble(artist) { bubble.className = 'similar-artist-bubble'; bubble.setAttribute('data-artist-id', artist.id); + // Track if image needs lazy loading + const hasImage = artist.image_url && artist.image_url.trim() !== ''; + bubble.setAttribute('data-needs-image', hasImage ? 'false' : 'true'); + // Create image container const imageContainer = document.createElement('div'); imageContainer.className = 'similar-artist-bubble-image'; - if (artist.image_url && artist.image_url.trim() !== '') { + if (hasImage) { const img = document.createElement('img'); img.src = artist.image_url; img.alt = artist.name; @@ -20219,11 +20428,12 @@ function createSimilarArtistBubble(artist) { img.onerror = () => { console.log(`Failed to load image for ${artist.name}`); imageContainer.innerHTML = `
🎡
`; + bubble.setAttribute('data-needs-image', 'true'); }; imageContainer.appendChild(img); } else { - // No image - show fallback + // No image - show fallback (will be lazy loaded) imageContainer.innerHTML = `
🎡
`; } From 1560726bbc226d8767a738de824e28351941b229 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Fri, 23 Jan 2026 14:25:26 -0800 Subject: [PATCH 06/20] rebuild discovery pool flow to allow multiprocessing of itunes and spotfiy. each getting their own pool. --- core/personalized_playlists.py | 201 +++++------ core/watchlist_scanner.py | 600 ++++++++++++++++++++------------- database/music_database.py | 251 ++++++++++---- web_server.py | 239 +++++++++---- webui/static/script.js | 20 +- 5 files changed, 856 insertions(+), 455 deletions(-) diff --git a/core/personalized_playlists.py b/core/personalized_playlists.py index 7bf2fd99..557ee394 100644 --- a/core/personalized_playlists.py +++ b/core/personalized_playlists.py @@ -100,6 +100,39 @@ class PersonalizedPlaylistsService: self.database = database self.spotify_client = spotify_client + def _get_active_source(self) -> str: + """ + Determine which music source is active for discovery. + Returns 'spotify' if Spotify is authenticated, 'itunes' otherwise. + """ + if self.spotify_client and hasattr(self.spotify_client, 'is_spotify_authenticated'): + if self.spotify_client.is_spotify_authenticated(): + return 'spotify' + return 'itunes' + + def _build_track_dict(self, row, source: str) -> Dict: + """Build a standardized track dictionary from a database row.""" + track_data = row['track_data_json'] + if isinstance(track_data, str): + try: + track_data = json.loads(track_data) + except: + track_data = None + + return { + 'track_id': row.get('spotify_track_id') or row.get('itunes_track_id'), + 'spotify_track_id': row.get('spotify_track_id'), + 'itunes_track_id': row.get('itunes_track_id'), + 'track_name': row['track_name'], + 'artist_name': row['artist_name'], + 'album_name': row['album_name'], + 'album_cover_url': row['album_cover_url'], + 'duration_ms': row['duration_ms'], + 'popularity': row.get('popularity', 0), + 'track_data_json': track_data, + 'source': source + } + @staticmethod def get_parent_genre(spotify_genre: str) -> str: """ @@ -166,25 +199,30 @@ class PersonalizedPlaylistsService: logger.error(f"Error getting forgotten favorites: {e}") return [] - def get_decade_playlist(self, decade: int, limit: int = 100) -> List[Dict]: + def get_decade_playlist(self, decade: int, limit: int = 100, source: str = None) -> List[Dict]: """ Get tracks from a specific decade from discovery pool with diversity filtering. Args: decade: Decade year (e.g., 2020 for 2020s, 2010 for 2010s) limit: Maximum tracks to return + source: Optional source filter ('spotify' or 'itunes'), auto-detects if not provided """ try: start_year = decade end_year = decade + 9 + # Determine active source if not specified + active_source = source or self._get_active_source() + with self.database._get_connection() as conn: cursor = conn.cursor() - # Query discovery_pool - get 10x more for diversity filtering + # Query discovery_pool - get 10x more for diversity filtering, filtered by source cursor.execute(""" SELECT spotify_track_id, + itunes_track_id, track_name, artist_name, album_name, @@ -192,26 +230,20 @@ class PersonalizedPlaylistsService: duration_ms, popularity, release_date, - track_data_json + track_data_json, + source FROM discovery_pool WHERE release_date IS NOT NULL AND CAST(SUBSTR(release_date, 1, 4) AS INTEGER) BETWEEN ? AND ? + AND source = ? ORDER BY RANDOM() LIMIT ? - """, (start_year, end_year, limit * 10)) + """, (start_year, end_year, active_source, limit * 10)) rows = cursor.fetchall() all_tracks = [] for row in rows: - track_dict = dict(row) - # Parse track_data_json if available - if track_dict.get('track_data_json'): - try: - import json - track_dict['track_data_json'] = json.loads(track_dict['track_data_json']) - except: - pass - all_tracks.append(track_dict) + all_tracks.append(self._build_track_dict(row, active_source)) if not all_tracks: logger.warning(f"No tracks found for {decade}s") @@ -268,22 +300,25 @@ class PersonalizedPlaylistsService: logger.error(f"Error getting decade playlist for {decade}s: {e}") return [] - def get_available_genres(self) -> List[Dict]: + def get_available_genres(self, source: str = None) -> List[Dict]: """ Get list of consolidated parent genres with track counts from discovery pool. Uses cached artist genres from database (populated during discovery scan). Consolidates specific Spotify genres into broader parent categories. """ try: + # Determine active source if not specified + active_source = source or self._get_active_source() + with self.database._get_connection() as conn: cursor = conn.cursor() - # Get all tracks with genres from discovery pool + # Get all tracks with genres from discovery pool, filtered by source cursor.execute(""" SELECT artist_genres FROM discovery_pool - WHERE artist_genres IS NOT NULL - """) + WHERE artist_genres IS NOT NULL AND source = ? + """, (active_source,)) rows = cursor.fetchall() if not rows: @@ -327,20 +362,24 @@ class PersonalizedPlaylistsService: logger.error(f"Error getting available genres: {e}") return [] - def get_genre_playlist(self, genre: str, limit: int = 50) -> List[Dict]: + def get_genre_playlist(self, genre: str, limit: int = 50, source: str = None) -> List[Dict]: """ Get tracks from a specific genre with diversity filtering. Uses cached artist genres from database (populated during discovery scan). Supports both parent genres (e.g., "Electronic/Dance") and specific genres (e.g., "house"). """ try: + # Determine active source if not specified + active_source = source or self._get_active_source() + with self.database._get_connection() as conn: cursor = conn.cursor() - # Get all tracks with genres from discovery pool + # Get all tracks with genres from discovery pool, filtered by source cursor.execute(""" SELECT spotify_track_id, + itunes_track_id, track_name, artist_name, album_name, @@ -348,10 +387,12 @@ class PersonalizedPlaylistsService: duration_ms, popularity, artist_genres, - track_data_json + track_data_json, + source FROM discovery_pool WHERE artist_genres IS NOT NULL - """) + AND source = ? + """, (active_source,)) rows = cursor.fetchall() # Determine if this is a parent genre or specific genre @@ -372,7 +413,7 @@ class PersonalizedPlaylistsService: for row in rows: try: - artist_genres_json = row[7] # artist_genres column + artist_genres_json = row['artist_genres'] if artist_genres_json: genres = json.loads(artist_genres_json) @@ -388,23 +429,7 @@ class PersonalizedPlaylistsService: break if genre_match: - # Convert row to dict (exclude artist_genres from output) - track_dict = { - 'spotify_track_id': row[0], - 'track_name': row[1], - 'artist_name': row[2], - 'album_name': row[3], - 'album_cover_url': row[4], - 'duration_ms': row[5], - 'popularity': row[6] - } - # Parse track_data_json if available - if row[8]: # track_data_json column - try: - track_dict['track_data_json'] = json.loads(row[8]) - except: - pass - matching_tracks.append(track_dict) + matching_tracks.append(self._build_track_dict(row, active_source)) except Exception as e: logger.debug(f"Error parsing genres for track: {e}") continue @@ -475,39 +500,34 @@ class PersonalizedPlaylistsService: def get_popular_picks(self, limit: int = 50) -> List[Dict]: """Get high popularity tracks from discovery pool with diversity (max 2 tracks per album/artist)""" + # Determine active source + active_source = self._get_active_source() + try: with self.database._get_connection() as conn: cursor = conn.cursor() - # Get more tracks than needed to allow for filtering + # Get more tracks than needed to allow for filtering, filtered by source cursor.execute(""" SELECT spotify_track_id, + itunes_track_id, track_name, artist_name, album_name, album_cover_url, duration_ms, popularity, - track_data_json + track_data_json, + source FROM discovery_pool - WHERE popularity >= 60 + WHERE popularity >= 60 AND source = ? ORDER BY popularity DESC, RANDOM() LIMIT ? - """, (limit * 3,)) # Get 3x more for diversity filtering + """, (active_source, limit * 3)) rows = cursor.fetchall() - all_tracks = [] - for row in rows: - track_dict = dict(row) - # Parse track_data_json if available - if track_dict.get('track_data_json'): - try: - import json - track_dict['track_data_json'] = json.loads(track_dict['track_data_json']) - except: - pass - all_tracks.append(track_dict) + all_tracks = [self._build_track_dict(row, active_source) for row in rows] # Apply diversity constraint: max 2 tracks per album, max 3 per artist tracks_by_album = {} @@ -531,7 +551,7 @@ class PersonalizedPlaylistsService: if len(diverse_tracks) >= limit: break - logger.info(f"Popular Picks: Selected {len(diverse_tracks)} tracks with diversity") + logger.info(f"Popular Picks ({active_source}): Selected {len(diverse_tracks)} tracks with diversity") return diverse_tracks[:limit] except Exception as e: @@ -540,6 +560,9 @@ class PersonalizedPlaylistsService: def get_hidden_gems(self, limit: int = 50) -> List[Dict]: """Get low popularity (underground/indie) tracks from discovery pool""" + # Determine active source + active_source = self._get_active_source() + try: with self.database._get_connection() as conn: cursor = conn.cursor() @@ -547,32 +570,23 @@ class PersonalizedPlaylistsService: cursor.execute(""" SELECT spotify_track_id, + itunes_track_id, track_name, artist_name, album_name, album_cover_url, duration_ms, popularity, - track_data_json + track_data_json, + source FROM discovery_pool - WHERE popularity < 40 + WHERE popularity < 40 AND source = ? ORDER BY RANDOM() LIMIT ? - """, (limit,)) + """, (active_source, limit)) rows = cursor.fetchall() - tracks = [] - for row in rows: - track_dict = dict(row) - # Parse track_data_json if available - if track_dict.get('track_data_json'): - try: - import json - track_dict['track_data_json'] = json.loads(track_dict['track_data_json']) - except: - pass - tracks.append(track_dict) - return tracks + return [self._build_track_dict(row, active_source) for row in rows] except Exception as e: logger.error(f"Error getting hidden gems: {e}") @@ -584,6 +598,9 @@ class PersonalizedPlaylistsService: Different every time you call it! """ + # Determine active source + active_source = self._get_active_source() + try: with self.database._get_connection() as conn: cursor = conn.cursor() @@ -591,31 +608,23 @@ class PersonalizedPlaylistsService: cursor.execute(""" SELECT spotify_track_id, + itunes_track_id, track_name, artist_name, album_name, album_cover_url, duration_ms, popularity, - track_data_json + track_data_json, + source FROM discovery_pool + WHERE source = ? ORDER BY RANDOM() LIMIT ? - """, (limit,)) + """, (active_source, limit)) rows = cursor.fetchall() - tracks = [] - for row in rows: - track_dict = dict(row) - # Parse track_data_json if available - if track_dict.get('track_data_json'): - try: - import json - track_dict['track_data_json'] = json.loads(track_dict['track_data_json']) - except: - pass - tracks.append(track_dict) - return tracks + return [self._build_track_dict(row, active_source) for row in rows] except Exception as e: logger.error(f"Error getting discovery shuffle: {e}") @@ -769,6 +778,9 @@ class PersonalizedPlaylistsService: def _get_discovery_tracks_by_category(self, category: str, limit: int) -> List[Dict]: """Get tracks from discovery pool matching genre or artist""" + # Determine active source + active_source = self._get_active_source() + try: with self.database._get_connection() as conn: cursor = conn.cursor() @@ -776,32 +788,23 @@ class PersonalizedPlaylistsService: cursor.execute(""" SELECT spotify_track_id, + itunes_track_id, track_name, artist_name, album_name, album_cover_url, duration_ms, popularity, - track_data_json + track_data_json, + source FROM discovery_pool - WHERE artist_name LIKE ? OR track_name LIKE ? + WHERE (artist_name LIKE ? OR track_name LIKE ?) AND source = ? ORDER BY RANDOM() LIMIT ? - """, (f'%{category}%', f'%{category}%', limit)) + """, (f'%{category}%', f'%{category}%', active_source, limit)) rows = cursor.fetchall() - tracks = [] - for row in rows: - track_dict = dict(row) - # Parse track_data_json if available - if track_dict.get('track_data_json'): - try: - import json - track_dict['track_data_json'] = json.loads(track_dict['track_data_json']) - except: - pass - tracks.append(track_dict) - return tracks + return [self._build_track_dict(row, active_source) for row in rows] except Exception as e: logger.error(f"Error getting discovery tracks by category: {e}") diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index d79962f3..35b65bb6 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -557,20 +557,18 @@ class WatchlistScanner: self.update_artist_scan_timestamp(db_artist_id) # Fetch and store similar artists for discovery feature (with caching to avoid over-polling) - # Note: Similar artists feature only works with Spotify - if provider == 'spotify' and watchlist_artist.spotify_artist_id: - try: - # Check if we have fresh similar artists cached (< 30 days old) - if self.database.has_fresh_similar_artists(watchlist_artist.spotify_artist_id, days_threshold=30): - logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping fetch") - else: - logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}...") - self.update_similar_artists(watchlist_artist) - logger.info(f"Similar artists updated for {watchlist_artist.artist_name}") - except Exception as similar_error: - logger.warning(f"Failed to update similar artists for {watchlist_artist.artist_name}: {similar_error}") - else: - logger.debug(f"Skipping similar artists for {watchlist_artist.artist_name} (iTunes doesn't support this feature)") + # Similar artists are fetched from MusicMap (works with any source) and matched to both Spotify and iTunes + source_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or str(watchlist_artist.id) + try: + # Check if we have fresh similar artists cached (< 30 days old) + if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30): + logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping fetch") + else: + logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}...") + self.update_similar_artists(watchlist_artist) + logger.info(f"Similar artists updated for {watchlist_artist.artist_name}") + except Exception as similar_error: + logger.warning(f"Failed to update similar artists for {watchlist_artist.artist_name}: {similar_error}") return ScanResult( artist_name=watchlist_artist.artist_name, @@ -1100,14 +1098,14 @@ class WatchlistScanner: def _fetch_similar_artists_from_musicmap(self, artist_name: str, limit: int = 20) -> List[Dict[str, Any]]: """ - Fetch similar artists from MusicMap and match them to Spotify. + Fetch similar artists from MusicMap and match them to both Spotify and iTunes. Args: artist_name: The artist name to find similar artists for limit: Maximum number of similar artists to return (default: 20) Returns: - List of matched artist dictionaries with Spotify data + List of matched artist dictionaries with both Spotify and iTunes IDs when available """ try: logger.info(f"Fetching similar artists from MusicMap for: {artist_name}") @@ -1151,52 +1149,94 @@ class WatchlistScanner: logger.info(f"Found {len(similar_artist_names)} similar artists from MusicMap") - # Get the searched artist's Spotify ID to exclude them - searched_artist_id = None - try: - searched_results = self.spotify_client.search_artists(artist_name, limit=1) - if searched_results and len(searched_results) > 0: - searched_artist_id = searched_results[0].id - except Exception as e: - logger.warning(f"Could not get searched artist ID: {e}") + # Get iTunes client for matching + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() - # Match each artist to Spotify + # Get the searched artist's IDs to exclude them + searched_spotify_id = None + searched_itunes_id = None + try: + # Try Spotify search + if self.spotify_client and self.spotify_client.is_spotify_authenticated(): + searched_results = self.spotify_client.search_artists(artist_name, limit=1) + if searched_results and len(searched_results) > 0: + searched_spotify_id = searched_results[0].id + except Exception as e: + logger.debug(f"Could not get searched artist Spotify ID: {e}") + + try: + # Try iTunes search + itunes_results = itunes_client.search_artists(artist_name, limit=1) + if itunes_results and len(itunes_results) > 0: + searched_itunes_id = itunes_results[0].id + except Exception as e: + logger.debug(f"Could not get searched artist iTunes ID: {e}") + + # Match each artist to both Spotify and iTunes matched_artists = [] - seen_artist_ids = set() # Track seen artist IDs to prevent duplicates + seen_names = set() # Track seen artist names to prevent duplicates for artist_name_to_match in similar_artist_names[:limit]: try: - # Search Spotify for the artist - results = self.spotify_client.search_artists(artist_name_to_match, limit=1) + # Skip if we've already matched this artist name + name_lower = artist_name_to_match.lower().strip() + if name_lower in seen_names: + continue - if results and len(results) > 0: - spotify_artist = results[0] + artist_data = { + 'name': artist_name_to_match, + 'spotify_id': None, + 'itunes_id': None, + 'image_url': None, + 'genres': [], + 'popularity': 0 + } - # Skip if this is the searched artist - if spotify_artist.id == searched_artist_id: - continue + # Try to match on Spotify + if self.spotify_client and self.spotify_client.is_spotify_authenticated(): + try: + spotify_results = self.spotify_client.search_artists(artist_name_to_match, limit=1) + if spotify_results and len(spotify_results) > 0: + spotify_artist = spotify_results[0] + # Skip if this is the searched artist + if spotify_artist.id != searched_spotify_id: + artist_data['spotify_id'] = spotify_artist.id + artist_data['name'] = spotify_artist.name # Use canonical name + artist_data['image_url'] = spotify_artist.image_url if hasattr(spotify_artist, 'image_url') else None + artist_data['genres'] = spotify_artist.genres if hasattr(spotify_artist, 'genres') else [] + artist_data['popularity'] = spotify_artist.popularity if hasattr(spotify_artist, 'popularity') else 0 + except Exception as e: + logger.debug(f"Spotify match failed for {artist_name_to_match}: {e}") - # Skip if we've already seen this artist ID (deduplication) - if spotify_artist.id in seen_artist_ids: - continue + # Try to match on iTunes + try: + itunes_results = itunes_client.search_artists(artist_name_to_match, limit=1) + if itunes_results and len(itunes_results) > 0: + itunes_artist = itunes_results[0] + # Skip if this is the searched artist + if itunes_artist.id != searched_itunes_id: + artist_data['itunes_id'] = itunes_artist.id + # Use iTunes name if we don't have Spotify + if not artist_data['spotify_id']: + artist_data['name'] = itunes_artist.name + # Use iTunes genres if we don't have Spotify genres + if not artist_data['genres'] and hasattr(itunes_artist, 'genres'): + artist_data['genres'] = itunes_artist.genres + except Exception as e: + logger.debug(f"iTunes match failed for {artist_name_to_match}: {e}") - seen_artist_ids.add(spotify_artist.id) - - matched_artists.append({ - 'id': spotify_artist.id, - 'name': spotify_artist.name, - 'image_url': spotify_artist.image_url if hasattr(spotify_artist, 'image_url') else None, - 'genres': spotify_artist.genres if hasattr(spotify_artist, 'genres') else [], - 'popularity': spotify_artist.popularity if hasattr(spotify_artist, 'popularity') else 0 - }) - - logger.debug(f" Matched: {spotify_artist.name}") + # Only add if we got at least one ID + if artist_data['spotify_id'] or artist_data['itunes_id']: + seen_names.add(name_lower) + matched_artists.append(artist_data) + logger.debug(f" Matched: {artist_data['name']} (Spotify: {artist_data['spotify_id']}, iTunes: {artist_data['itunes_id']})") except Exception as match_error: logger.debug(f"Error matching {artist_name_to_match}: {match_error}") continue - logger.info(f"Matched {len(matched_artists)} similar artists to Spotify") + logger.info(f"Matched {len(matched_artists)} similar artists (Spotify + iTunes)") return matched_artists except requests.exceptions.RequestException as e: @@ -1210,12 +1250,12 @@ class WatchlistScanner: """ Fetch and store similar artists for a watchlist artist. Called after each artist scan to build discovery pool. - Uses MusicMap to find similar artists and matches them to Spotify. + Uses MusicMap to find similar artists and matches them to both Spotify and iTunes. """ try: logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}") - # Get similar artists from MusicMap (returns list of artist dicts) + # Get similar artists from MusicMap (returns list of artist dicts with both IDs) similar_artists = self._fetch_similar_artists_from_musicmap(watchlist_artist.artist_name, limit=limit) if not similar_artists: @@ -1224,21 +1264,25 @@ class WatchlistScanner: logger.info(f"Found {len(similar_artists)} similar artists for {watchlist_artist.artist_name}") + # Use consistent source artist ID (prefer Spotify, fall back to iTunes or internal ID) + source_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or str(watchlist_artist.id) + # Store each similar artist in database stored_count = 0 for rank, similar_artist in enumerate(similar_artists, 1): try: - # similar_artist is a dict with 'id' and 'name' keys + # similar_artist has 'name', 'spotify_id', and 'itunes_id' keys success = self.database.add_or_update_similar_artist( - source_artist_id=watchlist_artist.spotify_artist_id, - similar_artist_spotify_id=similar_artist['id'], + source_artist_id=source_artist_id, similar_artist_name=similar_artist['name'], + similar_artist_spotify_id=similar_artist.get('spotify_id'), + similar_artist_itunes_id=similar_artist.get('itunes_id'), similarity_rank=rank ) if success: stored_count += 1 - logger.debug(f" #{rank}: {similar_artist['name']} (Spotify ID: {similar_artist['id']})") + logger.debug(f" #{rank}: {similar_artist['name']} (Spotify: {similar_artist.get('spotify_id')}, iTunes: {similar_artist.get('itunes_id')})") except Exception as e: logger.warning(f"Error storing similar artist {similar_artist.get('name', 'Unknown')}: {e}") @@ -1256,8 +1300,8 @@ class WatchlistScanner: Populate discovery pool with tracks from top similar artists. Called after watchlist scan completes. - IMPROVED: Larger pool for better discovery (50 artists x 10 releases = ~500 releases) - - Checks if pool was updated in last 24 hours (prevents over-polling Spotify) + Supports both Spotify and iTunes sources - populates for whichever is available. + - Checks if pool was updated in last 24 hours (prevents over-polling) - Includes albums, singles, and EPs for comprehensive coverage - Appends to existing pool instead of replacing it - Cleans up tracks older than 365 days (maintains 1 year rolling window) @@ -1266,13 +1310,27 @@ class WatchlistScanner: from datetime import datetime, timedelta import random - # Check if we should run (prevents over-polling Spotify) + # Check if we should run (prevents over-polling) if not self.database.should_populate_discovery_pool(hours_threshold=24): - logger.info("Discovery pool was populated recently (< 24 hours ago). Skipping to avoid over-polling Spotify.") + logger.info("Discovery pool was populated recently (< 24 hours ago). Skipping.") return logger.info("Populating discovery pool from similar artists...") + # Determine which sources are available + spotify_available = self.spotify_client and self.spotify_client.is_spotify_authenticated() + + # Import iTunes client for fallback + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() + itunes_available = True # iTunes is always available (no auth needed) + + if not spotify_available and not itunes_available: + logger.warning("No music sources available to populate discovery pool") + return + + logger.info(f"Sources available - Spotify: {spotify_available}, iTunes: {itunes_available}") + # Get top similar artists across all watchlist (ordered by occurrence_count) similar_artists = self.database.get_top_similar_artists(limit=top_artists_limit) @@ -1288,129 +1346,182 @@ class WatchlistScanner: try: logger.info(f"[{artist_idx}/{len(similar_artists)}] Processing {similar_artist.similar_artist_name} (occurrence: {similar_artist.occurrence_count})") - # Get artist's albums from Spotify - all_albums = self.spotify_client.get_artist_albums( - similar_artist.similar_artist_spotify_id, - album_type='album,single,ep', # Include albums, singles, and EPs for comprehensive discovery - limit=50 - ) + # Build list of sources to process for this artist + # iTunes is ALWAYS processed (baseline), Spotify is added if authenticated + sources_to_process = [] - if not all_albums: - logger.debug(f"No albums found for {similar_artist.similar_artist_name}") + # Always add iTunes first (baseline source) + if similar_artist.similar_artist_itunes_id: + sources_to_process.append(('itunes', similar_artist.similar_artist_itunes_id)) + + # Add Spotify if authenticated and we have an ID + if spotify_available and similar_artist.similar_artist_spotify_id: + sources_to_process.append(('spotify', similar_artist.similar_artist_spotify_id)) + + if not sources_to_process: + logger.debug(f"No valid IDs for {similar_artist.similar_artist_name}, skipping") continue - # Fetch artist genres once for all tracks of this artist - artist_genres = [] - try: - artist_data = self.spotify_client.get_artist(similar_artist.similar_artist_spotify_id) - if artist_data and 'genres' in artist_data: - artist_genres = artist_data['genres'] - except Exception as e: - logger.debug(f"Could not fetch genres for {similar_artist.similar_artist_name}: {e}") + logger.debug(f" Processing {len(sources_to_process)} source(s): {[s[0] for s in sources_to_process]}") - # IMPROVED: Smart selection mixing albums, singles, and EPs - # Prioritize recent releases and popular content - - # Separate by type for balanced selection - albums = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type == 'album'] - singles_eps = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type in ['single', 'ep']] - other = [a for a in all_albums if not hasattr(a, 'album_type')] - - # Select albums: latest releases + popular older content - selected_albums = [] - - # Always include 3 most recent releases (any type) - this captures new singles/EPs - latest_releases = all_albums[:3] - selected_albums.extend(latest_releases) - - # Add remaining slots with balanced mix - remaining_slots = albums_per_artist - len(selected_albums) - if remaining_slots > 0: - # Combine remaining albums and singles - remaining_content = all_albums[3:] - - if len(remaining_content) > remaining_slots: - # Randomly select from remaining content - random_selection = random.sample(remaining_content, remaining_slots) - selected_albums.extend(random_selection) - else: - selected_albums.extend(remaining_content) - - logger.info(f" Selected {len(selected_albums)} releases from {len(all_albums)} available (albums: {len(albums)}, singles/EPs: {len(singles_eps)})") - - # Process each selected album - for album_idx, album in enumerate(selected_albums, 1): + # Process each source for this artist + for source, artist_id in sources_to_process: try: - # Get full album data with tracks - album_data = self.spotify_client.get_album(album.id) + # Get artist's albums from this source + if source == 'spotify': + all_albums = self.spotify_client.get_artist_albums( + artist_id, + album_type='album,single,ep', + limit=50 + ) + else: # itunes + all_albums = itunes_client.get_artist_albums( + artist_id, + album_type='album,single', + limit=50 + ) - if not album_data or 'tracks' not in album_data: + if not all_albums: + logger.debug(f"No albums found for {similar_artist.similar_artist_name} on {source}") continue - tracks = album_data['tracks'].get('items', []) - logger.debug(f" Album {album_idx}: {album_data.get('name', 'Unknown')} ({len(tracks)} tracks)") - - # Determine if this is a new release (within last 30 days) - is_new = False + # Fetch artist genres for this source + artist_genres = [] try: - release_date_str = album_data.get('release_date', '') - if release_date_str: - if len(release_date_str) == 10: # Full date - release_date = datetime.strptime(release_date_str, "%Y-%m-%d") - days_old = (datetime.now() - release_date).days - is_new = days_old <= 30 - except: - pass + if source == 'spotify': + artist_data = self.spotify_client.get_artist(artist_id) + if artist_data and 'genres' in artist_data: + artist_genres = artist_data['genres'] + else: # iTunes - genres from artist lookup + artist_data = itunes_client.get_artist(artist_id) + if artist_data and 'genres' in artist_data: + artist_genres = artist_data['genres'] + except Exception as e: + logger.debug(f"Could not fetch genres for {similar_artist.similar_artist_name} on {source}: {e}") - # Add each track to discovery pool - for track in tracks: + # IMPROVED: Smart selection mixing albums, singles, and EPs + # Prioritize recent releases and popular content + + # Separate by type for balanced selection + albums = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type == 'album'] + singles_eps = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type in ['single', 'ep']] + other = [a for a in all_albums if not hasattr(a, 'album_type')] + + # Select albums: latest releases + popular older content + selected_albums = [] + + # Always include 3 most recent releases (any type) - this captures new singles/EPs + latest_releases = all_albums[:3] + selected_albums.extend(latest_releases) + + # Add remaining slots with balanced mix + remaining_slots = albums_per_artist - len(selected_albums) + if remaining_slots > 0: + # Combine remaining albums and singles + remaining_content = all_albums[3:] + + if len(remaining_content) > remaining_slots: + # Randomly select from remaining content + random_selection = random.sample(remaining_content, remaining_slots) + selected_albums.extend(random_selection) + else: + selected_albums.extend(remaining_content) + + logger.info(f" [{source}] Selected {len(selected_albums)} releases from {len(all_albums)} available (albums: {len(albums)}, singles/EPs: {len(singles_eps)})") + + # Process each selected album + for album_idx, album in enumerate(selected_albums, 1): try: - # Enhance track object with full album data (including album_type) - enhanced_track = { - **track, - 'album': { - 'id': album_data['id'], - 'name': album_data.get('name', 'Unknown Album'), - 'images': album_data.get('images', []), - 'release_date': album_data.get('release_date', ''), - 'album_type': album_data.get('album_type', 'album'), - 'total_tracks': album_data.get('total_tracks', 0) - } - } + # Get full album data with tracks from appropriate source + if source == 'spotify': + album_data = self.spotify_client.get_album(album.id) + if not album_data or 'tracks' not in album_data: + continue + tracks = album_data['tracks'].get('items', []) + else: # itunes + album_data = itunes_client.get_album(album.id) + if not album_data: + continue + # iTunes get_album doesn't include tracks inline, need separate call + tracks_data = itunes_client.get_album_tracks(album.id) + tracks = tracks_data.get('items', []) if tracks_data else [] - # Build track data for discovery pool - track_data = { - 'spotify_track_id': track['id'], - 'spotify_album_id': album_data['id'], - 'spotify_artist_id': similar_artist.similar_artist_spotify_id, - 'track_name': track['name'], - 'artist_name': similar_artist.similar_artist_name, - 'album_name': album_data.get('name', 'Unknown Album'), - 'album_cover_url': album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None, - 'duration_ms': track.get('duration_ms', 0), - 'popularity': album_data.get('popularity', 0), - 'release_date': album_data.get('release_date', ''), - 'is_new_release': is_new, - 'track_data_json': enhanced_track, # Store enhanced track with full album data - 'artist_genres': artist_genres # Add cached genres - } + logger.debug(f" Album {album_idx}: {album_data.get('name', 'Unknown')} ({len(tracks)} tracks)") - # Add to discovery pool - if self.database.add_to_discovery_pool(track_data): - total_tracks_added += 1 + # Determine if this is a new release (within last 30 days) + is_new = False + try: + release_date_str = album_data.get('release_date', '') + if release_date_str: + # Handle full date or year-only + if len(release_date_str) >= 10: + release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d") + days_old = (datetime.now() - release_date).days + is_new = days_old <= 30 + except: + pass - except Exception as track_error: - logger.debug(f"Error adding track to discovery pool: {track_error}") + # Add each track to discovery pool + for track in tracks: + try: + # Enhance track object with full album data (including album_type) + enhanced_track = { + **track, + 'album': { + 'id': album_data['id'], + 'name': album_data.get('name', 'Unknown Album'), + 'images': album_data.get('images', []), + 'release_date': album_data.get('release_date', ''), + 'album_type': album_data.get('album_type', 'album'), + 'total_tracks': album_data.get('total_tracks', 0) + }, + '_source': source + } + + # Build track data for discovery pool with source-specific IDs + track_data = { + 'track_name': track.get('name', 'Unknown Track'), + 'artist_name': similar_artist.similar_artist_name, + 'album_name': album_data.get('name', 'Unknown Album'), + 'album_cover_url': album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None, + 'duration_ms': track.get('duration_ms', 0), + 'popularity': album_data.get('popularity', 0), + 'release_date': album_data.get('release_date', ''), + 'is_new_release': is_new, + 'track_data_json': enhanced_track, + 'artist_genres': artist_genres + } + + # Add source-specific IDs + if source == 'spotify': + track_data['spotify_track_id'] = track.get('id') + track_data['spotify_album_id'] = album_data.get('id') + track_data['spotify_artist_id'] = similar_artist.similar_artist_spotify_id + else: # itunes + track_data['itunes_track_id'] = track.get('id') + track_data['itunes_album_id'] = album_data.get('id') + track_data['itunes_artist_id'] = similar_artist.similar_artist_itunes_id + + # Add to discovery pool with source + if self.database.add_to_discovery_pool(track_data, source=source): + total_tracks_added += 1 + + except Exception as track_error: + logger.debug(f"Error adding track to discovery pool: {track_error}") + continue + + # Small delay between albums + time.sleep(DELAY_BETWEEN_ALBUMS) + + except Exception as album_error: + logger.warning(f"Error processing album on {source}: {album_error}") continue - # Small delay between albums - time.sleep(DELAY_BETWEEN_ALBUMS) - - except Exception as album_error: - logger.warning(f"Error processing album: {album_error}") + except Exception as source_error: + logger.warning(f"Error processing {source} source for {similar_artist.similar_artist_name}: {source_error}") continue - # Delay between artists + # Delay between artists (after processing all sources for this artist) if artist_idx < len(similar_artists): time.sleep(DELAY_BETWEEN_ARTISTS) @@ -1441,76 +1552,115 @@ class WatchlistScanner: for db_idx, album_row in enumerate(db_albums, 1): try: - # Search for album on Spotify - query = f"album:{album_row['title']} artist:{album_row['artist_name']}" - search_results = self.spotify_client.search_albums(query, limit=1) + query = f"{album_row['title']} {album_row['artist_name']}" + album_data = None + tracks = [] + db_source = None + artist_id_for_genres = None - if search_results and len(search_results) > 0: - spotify_album = search_results[0] - album_data = self.spotify_client.get_album(spotify_album.id) + # Try Spotify first if available + if spotify_available: + try: + search_results = self.spotify_client.search_albums(f"album:{album_row['title']} artist:{album_row['artist_name']}", limit=1) + if search_results and len(search_results) > 0: + spotify_album = search_results[0] + album_data = self.spotify_client.get_album(spotify_album.id) + if album_data and 'tracks' in album_data: + tracks = album_data['tracks'].get('items', []) + db_source = 'spotify' + if album_data.get('artists'): + artist_id_for_genres = album_data['artists'][0]['id'] + except Exception as e: + logger.debug(f"Spotify search failed for {album_row['title']}: {e}") - if album_data and 'tracks' in album_data: - tracks = album_data['tracks'].get('items', []) + # Fall back to iTunes if Spotify didn't work + if not tracks and itunes_available: + try: + search_results = itunes_client.search_albums(query, limit=1) + if search_results and len(search_results) > 0: + itunes_album = search_results[0] + album_data = itunes_client.get_album(itunes_album.id) + if album_data: + tracks_data = itunes_client.get_album_tracks(itunes_album.id) + tracks = tracks_data.get('items', []) if tracks_data else [] + db_source = 'itunes' + # For iTunes, artist ID is in the album data + if album_data.get('artists'): + artist_id_for_genres = album_data['artists'][0].get('id') + except Exception as e: + logger.debug(f"iTunes search failed for {album_row['title']}: {e}") - # Fetch artist genres - artist_genres = [] - try: - if album_data.get('artists') and len(album_data['artists']) > 0: - artist_id = album_data['artists'][0]['id'] - artist_data = self.spotify_client.get_artist(artist_id) - if artist_data and 'genres' in artist_data: - artist_genres = artist_data['genres'] - except Exception as e: - logger.debug(f"Could not fetch genres for album artist: {e}") + if not tracks or not album_data: + continue - # Check if new release - is_new = False - try: - release_date_str = album_data.get('release_date', '') - if release_date_str and len(release_date_str) == 10: - release_date = datetime.strptime(release_date_str, "%Y-%m-%d") - days_old = (datetime.now() - release_date).days - is_new = days_old <= 30 - except: - pass + # Fetch artist genres + artist_genres = [] + try: + if artist_id_for_genres: + if db_source == 'spotify': + artist_data = self.spotify_client.get_artist(artist_id_for_genres) + else: + artist_data = itunes_client.get_artist(artist_id_for_genres) + if artist_data and 'genres' in artist_data: + artist_genres = artist_data['genres'] + except Exception as e: + logger.debug(f"Could not fetch genres for album artist: {e}") - for track in tracks: - try: - # Enhance track object with full album data (including album_type) - enhanced_track = { - **track, - 'album': { - 'id': album_data['id'], - 'name': album_row['title'], - 'images': album_data.get('images', []), - 'release_date': album_data.get('release_date', ''), - 'album_type': album_data.get('album_type', 'album'), - 'total_tracks': album_data.get('total_tracks', 0) - } - } + # Check if new release + is_new = False + try: + release_date_str = album_data.get('release_date', '') + if release_date_str and len(release_date_str) >= 10: + release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d") + days_old = (datetime.now() - release_date).days + is_new = days_old <= 30 + except: + pass - track_data = { - 'spotify_track_id': track['id'], - 'spotify_album_id': album_data['id'], - 'spotify_artist_id': album_data['artists'][0]['id'] if album_data.get('artists') else '', - 'track_name': track['name'], - 'artist_name': album_row['artist_name'], - 'album_name': album_row['title'], - 'album_cover_url': album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None, - 'duration_ms': track.get('duration_ms', 0), - 'popularity': album_data.get('popularity', 0), - 'release_date': album_data.get('release_date', ''), - 'is_new_release': is_new, - 'track_data_json': enhanced_track, # Store enhanced track with full album data - 'artist_genres': artist_genres - } + for track in tracks: + try: + enhanced_track = { + **track, + 'album': { + 'id': album_data['id'], + 'name': album_row['title'], + 'images': album_data.get('images', []), + 'release_date': album_data.get('release_date', ''), + 'album_type': album_data.get('album_type', 'album'), + 'total_tracks': album_data.get('total_tracks', 0) + }, + '_source': db_source + } - if self.database.add_to_discovery_pool(track_data): - total_tracks_added += 1 - except Exception as track_error: - continue + track_data = { + 'track_name': track.get('name', 'Unknown Track'), + 'artist_name': album_row['artist_name'], + 'album_name': album_row['title'], + 'album_cover_url': album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None, + 'duration_ms': track.get('duration_ms', 0), + 'popularity': album_data.get('popularity', 0), + 'release_date': album_data.get('release_date', ''), + 'is_new_release': is_new, + 'track_data_json': enhanced_track, + 'artist_genres': artist_genres + } - time.sleep(DELAY_BETWEEN_ALBUMS) + # Add source-specific IDs + if db_source == 'spotify': + track_data['spotify_track_id'] = track.get('id') + track_data['spotify_album_id'] = album_data.get('id') + track_data['spotify_artist_id'] = artist_id_for_genres or '' + else: # itunes + track_data['itunes_track_id'] = track.get('id') + track_data['itunes_album_id'] = album_data.get('id') + track_data['itunes_artist_id'] = artist_id_for_genres or '' + + if self.database.add_to_discovery_pool(track_data, source=db_source): + total_tracks_added += 1 + except Exception as track_error: + continue + + time.sleep(DELAY_BETWEEN_ALBUMS) except Exception as album_error: logger.debug(f"Error processing database album {album_row['title']}: {album_error}") continue diff --git a/database/music_database.py b/database/music_database.py index 2f774d97..9d198219 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -97,10 +97,11 @@ class WatchlistArtist: @dataclass class SimilarArtist: - """Similar artist recommendation from Spotify""" + """Similar artist recommendation from Spotify/iTunes""" id: int source_artist_id: str # Watchlist artist's database ID - similar_artist_spotify_id: str + similar_artist_spotify_id: Optional[str] # Spotify artist ID (may be None if iTunes-only) + similar_artist_itunes_id: Optional[str] # iTunes artist ID (may be None if Spotify-only) similar_artist_name: str similarity_rank: int # 1-10, where 1 is most similar occurrence_count: int # How many watchlist artists share this similar artist @@ -110,9 +111,13 @@ class SimilarArtist: class DiscoveryTrack: """Track in the discovery pool for recommendations""" id: int - spotify_track_id: str - spotify_album_id: str - spotify_artist_id: str + spotify_track_id: Optional[str] # Spotify track ID (None if iTunes source) + spotify_album_id: Optional[str] # Spotify album ID (None if iTunes source) + spotify_artist_id: Optional[str] # Spotify artist ID (None if iTunes source) + itunes_track_id: Optional[str] # iTunes track ID (None if Spotify source) + itunes_album_id: Optional[str] # iTunes album ID (None if Spotify source) + itunes_artist_id: Optional[str] # iTunes artist ID (None if Spotify source) + source: str # 'spotify' or 'itunes' track_name: str artist_name: str album_name: str @@ -121,7 +126,7 @@ class DiscoveryTrack: popularity: int release_date: str is_new_release: bool # Released within last 30 days - track_data_json: str # Full Spotify track object for modal + track_data_json: str # Full track object for modal (Spotify or iTunes format) added_date: datetime @dataclass @@ -129,7 +134,9 @@ class RecentRelease: """Recent album release from watchlist artist""" id: int watchlist_artist_id: int - album_spotify_id: str + album_spotify_id: Optional[str] # Spotify album ID (None if iTunes source) + album_itunes_id: Optional[str] # iTunes album ID (None if Spotify source) + source: str # 'spotify' or 'itunes' album_name: str release_date: str album_cover_url: Optional[str] @@ -450,26 +457,33 @@ class MusicDatabase: """Add tables for discovery feature: similar artists, discovery pool, and recent releases""" try: # Similar Artists table - stores similar artists for each watchlist artist + # Supports both Spotify and iTunes IDs for dual-source discovery cursor.execute(""" CREATE TABLE IF NOT EXISTS similar_artists ( id INTEGER PRIMARY KEY AUTOINCREMENT, source_artist_id TEXT NOT NULL, - similar_artist_spotify_id TEXT NOT NULL, + similar_artist_spotify_id TEXT, + similar_artist_itunes_id TEXT, similar_artist_name TEXT NOT NULL, similarity_rank INTEGER DEFAULT 1, occurrence_count INTEGER DEFAULT 1, last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(source_artist_id, similar_artist_spotify_id) + UNIQUE(source_artist_id, similar_artist_name) ) """) # Discovery Pool table - rotating pool of 1000-2000 tracks for recommendations + # Supports both Spotify and iTunes sources for dual-source discovery cursor.execute(""" CREATE TABLE IF NOT EXISTS discovery_pool ( id INTEGER PRIMARY KEY AUTOINCREMENT, - spotify_track_id TEXT UNIQUE NOT NULL, - spotify_album_id TEXT NOT NULL, - spotify_artist_id TEXT NOT NULL, + spotify_track_id TEXT, + spotify_album_id TEXT, + spotify_artist_id TEXT, + itunes_track_id TEXT, + itunes_album_id TEXT, + itunes_artist_id TEXT, + source TEXT NOT NULL DEFAULT 'spotify', track_name TEXT NOT NULL, artist_name TEXT NOT NULL, album_name TEXT NOT NULL, @@ -479,38 +493,47 @@ class MusicDatabase: release_date TEXT, is_new_release BOOLEAN DEFAULT 0, track_data_json TEXT NOT NULL, - added_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP + added_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(spotify_track_id, itunes_track_id, source) ) """) # Recent Releases table - tracks new releases from watchlist artists + # Supports both Spotify and iTunes sources for dual-source discovery cursor.execute(""" CREATE TABLE IF NOT EXISTS recent_releases ( id INTEGER PRIMARY KEY AUTOINCREMENT, watchlist_artist_id INTEGER NOT NULL, - album_spotify_id TEXT NOT NULL, + album_spotify_id TEXT, + album_itunes_id TEXT, + source TEXT NOT NULL DEFAULT 'spotify', album_name TEXT NOT NULL, release_date TEXT NOT NULL, album_cover_url TEXT, track_count INTEGER DEFAULT 0, added_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(watchlist_artist_id, album_spotify_id), + UNIQUE(watchlist_artist_id, album_spotify_id, album_itunes_id), FOREIGN KEY (watchlist_artist_id) REFERENCES watchlist_artists (id) ON DELETE CASCADE ) """) # Discovery Recent Albums cache - for discover page recent releases section + # Supports both Spotify and iTunes sources for dual-source discovery cursor.execute(""" CREATE TABLE IF NOT EXISTS discovery_recent_albums ( id INTEGER PRIMARY KEY AUTOINCREMENT, - album_spotify_id TEXT NOT NULL UNIQUE, + album_spotify_id TEXT, + album_itunes_id TEXT, + artist_spotify_id TEXT, + artist_itunes_id TEXT, + source TEXT NOT NULL DEFAULT 'spotify', album_name TEXT NOT NULL, artist_name TEXT NOT NULL, - artist_spotify_id TEXT NOT NULL, album_cover_url TEXT, release_date TEXT NOT NULL, album_type TEXT DEFAULT 'album', - cached_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP + cached_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(album_spotify_id, album_itunes_id, source) ) """) @@ -571,13 +594,20 @@ class MusicDatabase: # Create indexes for performance cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_source ON similar_artists (source_artist_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_spotify ON similar_artists (similar_artist_spotify_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_itunes ON similar_artists (similar_artist_itunes_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_occurrence ON similar_artists (occurrence_count)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_name ON similar_artists (similar_artist_name)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_spotify_track ON discovery_pool (spotify_track_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_itunes_track ON discovery_pool (itunes_track_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_artist ON discovery_pool (spotify_artist_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_itunes_artist ON discovery_pool (itunes_artist_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_source ON discovery_pool (source)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_added_date ON discovery_pool (added_date)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_is_new ON discovery_pool (is_new_release)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_watchlist ON recent_releases (watchlist_artist_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_date ON recent_releases (release_date)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_source ON recent_releases (source)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_source ON discovery_recent_albums (source)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_date ON discovery_recent_albums (release_date)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_playlists_type ON listenbrainz_playlists (playlist_type)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_playlists_mbid ON listenbrainz_playlists (playlist_mbid)") @@ -593,6 +623,41 @@ class MusicDatabase: cursor.execute("ALTER TABLE discovery_pool ADD COLUMN artist_genres TEXT") logger.info("Added artist_genres column to discovery_pool table") + # Migration: Add iTunes columns to discovery_pool for dual-source discovery + if 'itunes_track_id' not in discovery_pool_columns: + cursor.execute("ALTER TABLE discovery_pool ADD COLUMN itunes_track_id TEXT") + cursor.execute("ALTER TABLE discovery_pool ADD COLUMN itunes_album_id TEXT") + cursor.execute("ALTER TABLE discovery_pool ADD COLUMN itunes_artist_id TEXT") + cursor.execute("ALTER TABLE discovery_pool ADD COLUMN source TEXT DEFAULT 'spotify'") + logger.info("Added iTunes columns to discovery_pool table for dual-source discovery") + + # Migration: Add iTunes ID to similar_artists for dual-source discovery + cursor.execute("PRAGMA table_info(similar_artists)") + similar_artists_columns = [column[1] for column in cursor.fetchall()] + + if 'similar_artist_itunes_id' not in similar_artists_columns: + cursor.execute("ALTER TABLE similar_artists ADD COLUMN similar_artist_itunes_id TEXT") + logger.info("Added similar_artist_itunes_id column to similar_artists table") + + # Migration: Add iTunes columns to recent_releases for dual-source discovery + cursor.execute("PRAGMA table_info(recent_releases)") + recent_releases_columns = [column[1] for column in cursor.fetchall()] + + if 'album_itunes_id' not in recent_releases_columns: + cursor.execute("ALTER TABLE recent_releases ADD COLUMN album_itunes_id TEXT") + cursor.execute("ALTER TABLE recent_releases ADD COLUMN source TEXT DEFAULT 'spotify'") + logger.info("Added iTunes columns to recent_releases table for dual-source discovery") + + # Migration: Add iTunes columns to discovery_recent_albums for dual-source discovery + cursor.execute("PRAGMA table_info(discovery_recent_albums)") + discovery_recent_albums_columns = [column[1] for column in cursor.fetchall()] + + if 'album_itunes_id' not in discovery_recent_albums_columns: + cursor.execute("ALTER TABLE discovery_recent_albums ADD COLUMN album_itunes_id TEXT") + cursor.execute("ALTER TABLE discovery_recent_albums ADD COLUMN artist_itunes_id TEXT") + cursor.execute("ALTER TABLE discovery_recent_albums ADD COLUMN source TEXT DEFAULT 'spotify'") + logger.info("Added iTunes columns to discovery_recent_albums table for dual-source discovery") + logger.info("Discovery tables created successfully") except Exception as e: @@ -2933,23 +2998,28 @@ class MusicDatabase: # === Discovery Feature Methods === - def add_or_update_similar_artist(self, source_artist_id: str, similar_artist_spotify_id: str, - similar_artist_name: str, similarity_rank: int = 1) -> bool: - """Add or update a similar artist recommendation""" + def add_or_update_similar_artist(self, source_artist_id: str, similar_artist_name: str, + similar_artist_spotify_id: Optional[str] = None, + similar_artist_itunes_id: Optional[str] = None, + similarity_rank: int = 1) -> bool: + """Add or update a similar artist recommendation (supports both Spotify and iTunes IDs)""" try: with self._get_connection() as conn: cursor = conn.cursor() + # Use artist name as the unique key (allows storing both IDs for same artist) cursor.execute(""" INSERT INTO similar_artists - (source_artist_id, similar_artist_spotify_id, similar_artist_name, similarity_rank, occurrence_count, last_updated) - VALUES (?, ?, ?, ?, 1, CURRENT_TIMESTAMP) - ON CONFLICT(source_artist_id, similar_artist_spotify_id) + (source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, similar_artist_name, similarity_rank, occurrence_count, last_updated) + VALUES (?, ?, ?, ?, ?, 1, CURRENT_TIMESTAMP) + ON CONFLICT(source_artist_id, similar_artist_name) DO UPDATE SET + similar_artist_spotify_id = COALESCE(excluded.similar_artist_spotify_id, similar_artist_spotify_id), + similar_artist_itunes_id = COALESCE(excluded.similar_artist_itunes_id, similar_artist_itunes_id), similarity_rank = excluded.similarity_rank, occurrence_count = occurrence_count + 1, last_updated = CURRENT_TIMESTAMP - """, (source_artist_id, similar_artist_spotify_id, similar_artist_name, similarity_rank)) + """, (source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, similar_artist_name, similarity_rank)) conn.commit() return True @@ -2975,6 +3045,7 @@ class MusicDatabase: id=row['id'], source_artist_id=row['source_artist_id'], similar_artist_spotify_id=row['similar_artist_spotify_id'], + similar_artist_itunes_id=row['similar_artist_itunes_id'] if 'similar_artist_itunes_id' in row.keys() else None, similar_artist_name=row['similar_artist_name'], similarity_rank=row['similarity_rank'], occurrence_count=row['occurrence_count'], @@ -3026,13 +3097,14 @@ class MusicDatabase: SELECT MAX(id) as id, MAX(source_artist_id) as source_artist_id, - similar_artist_spotify_id, + MAX(similar_artist_spotify_id) as similar_artist_spotify_id, + MAX(similar_artist_itunes_id) as similar_artist_itunes_id, similar_artist_name, AVG(similarity_rank) as similarity_rank, SUM(occurrence_count) as occurrence_count, MAX(last_updated) as last_updated FROM similar_artists - GROUP BY similar_artist_spotify_id, similar_artist_name + GROUP BY similar_artist_name ORDER BY occurrence_count DESC, similarity_rank ASC LIMIT ? """, (limit,)) @@ -3042,6 +3114,7 @@ class MusicDatabase: id=row['id'], source_artist_id=row['source_artist_id'], similar_artist_spotify_id=row['similar_artist_spotify_id'], + similar_artist_itunes_id=row['similar_artist_itunes_id'] if 'similar_artist_itunes_id' in row.keys() else None, similar_artist_name=row['similar_artist_name'], similarity_rank=int(row['similarity_rank']), occurrence_count=row['occurrence_count'], @@ -3052,15 +3125,24 @@ class MusicDatabase: logger.error(f"Error getting top similar artists: {e}") return [] - def add_to_discovery_pool(self, track_data: Dict[str, Any]) -> bool: - """Add a track to the discovery pool""" + def add_to_discovery_pool(self, track_data: Dict[str, Any], source: str = 'spotify') -> bool: + """Add a track to the discovery pool (supports both Spotify and iTunes sources)""" try: with self._get_connection() as conn: cursor = conn.cursor() - # Check if track already exists - cursor.execute("SELECT COUNT(*) as count FROM discovery_pool WHERE spotify_track_id = ?", - (track_data['spotify_track_id'],)) + # Check if track already exists based on source + if source == 'spotify' and track_data.get('spotify_track_id'): + cursor.execute("SELECT COUNT(*) as count FROM discovery_pool WHERE spotify_track_id = ? AND source = 'spotify'", + (track_data['spotify_track_id'],)) + elif source == 'itunes' and track_data.get('itunes_track_id'): + cursor.execute("SELECT COUNT(*) as count FROM discovery_pool WHERE itunes_track_id = ? AND source = 'itunes'", + (track_data['itunes_track_id'],)) + else: + # Fallback check by track name and artist + cursor.execute("SELECT COUNT(*) as count FROM discovery_pool WHERE track_name = ? AND artist_name = ? AND source = ?", + (track_data['track_name'], track_data['artist_name'], source)) + if cursor.fetchone()['count'] > 0: return True # Already in pool @@ -3070,14 +3152,19 @@ class MusicDatabase: cursor.execute(""" INSERT INTO discovery_pool - (spotify_track_id, spotify_album_id, spotify_artist_id, track_name, artist_name, - album_name, album_cover_url, duration_ms, popularity, release_date, - is_new_release, track_data_json, artist_genres, added_date) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + (spotify_track_id, spotify_album_id, spotify_artist_id, + itunes_track_id, itunes_album_id, itunes_artist_id, + source, track_name, artist_name, album_name, album_cover_url, + duration_ms, popularity, release_date, is_new_release, track_data_json, artist_genres, added_date) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) """, ( - track_data['spotify_track_id'], - track_data['spotify_album_id'], - track_data['spotify_artist_id'], + track_data.get('spotify_track_id'), + track_data.get('spotify_album_id'), + track_data.get('spotify_artist_id'), + track_data.get('itunes_track_id'), + track_data.get('itunes_album_id'), + track_data.get('itunes_artist_id'), + source, track_data['track_name'], track_data['artist_name'], track_data['album_name'], @@ -3124,32 +3211,45 @@ class MusicDatabase: except Exception as e: logger.error(f"Error rotating discovery pool: {e}") - def get_discovery_pool_tracks(self, limit: int = 100, new_releases_only: bool = False) -> List[DiscoveryTrack]: - """Get tracks from discovery pool""" + def get_discovery_pool_tracks(self, limit: int = 100, new_releases_only: bool = False, source: Optional[str] = None) -> List[DiscoveryTrack]: + """Get tracks from discovery pool, optionally filtered by source ('spotify' or 'itunes')""" try: with self._get_connection() as conn: cursor = conn.cursor() + # Build query with optional source filter + where_clauses = [] + params = [] + if new_releases_only: - cursor.execute(""" - SELECT * FROM discovery_pool - WHERE is_new_release = 1 - ORDER BY added_date DESC - LIMIT ? - """, (limit,)) - else: - cursor.execute(""" - SELECT * FROM discovery_pool - ORDER BY added_date DESC - LIMIT ? - """, (limit,)) + where_clauses.append("is_new_release = 1") + + if source: + where_clauses.append("source = ?") + params.append(source) + + where_sql = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else "" + params.append(limit) + + cursor.execute(f""" + SELECT * FROM discovery_pool + {where_sql} + ORDER BY added_date DESC + LIMIT ? + """, params) rows = cursor.fetchall() + row_keys = rows[0].keys() if rows else [] + return [DiscoveryTrack( id=row['id'], spotify_track_id=row['spotify_track_id'], spotify_album_id=row['spotify_album_id'], spotify_artist_id=row['spotify_artist_id'], + itunes_track_id=row['itunes_track_id'] if 'itunes_track_id' in row_keys else None, + itunes_album_id=row['itunes_album_id'] if 'itunes_album_id' in row_keys else None, + itunes_artist_id=row['itunes_artist_id'] if 'itunes_artist_id' in row_keys else None, + source=row['source'] if 'source' in row_keys else 'spotify', track_name=row['track_name'], artist_name=row['artist_name'], album_name=row['album_name'], @@ -3166,21 +3266,25 @@ class MusicDatabase: logger.error(f"Error getting discovery pool tracks: {e}") return [] - def cache_discovery_recent_album(self, album_data: Dict[str, Any]) -> bool: - """Cache a recent album for the discover page (from watchlist or similar artists)""" + def cache_discovery_recent_album(self, album_data: Dict[str, Any], source: str = 'spotify') -> bool: + """Cache a recent album for the discover page (supports both Spotify and iTunes sources)""" try: with self._get_connection() as conn: cursor = conn.cursor() cursor.execute(""" INSERT OR REPLACE INTO discovery_recent_albums - (album_spotify_id, album_name, artist_name, artist_spotify_id, album_cover_url, release_date, album_type, cached_date) - VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + (album_spotify_id, album_itunes_id, artist_spotify_id, artist_itunes_id, source, + album_name, artist_name, album_cover_url, release_date, album_type, cached_date) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) """, ( - album_data['album_spotify_id'], + album_data.get('album_spotify_id'), + album_data.get('album_itunes_id'), + album_data.get('artist_spotify_id'), + album_data.get('artist_itunes_id'), + source, album_data['album_name'], album_data['artist_name'], - album_data['artist_spotify_id'], album_data.get('album_cover_url'), album_data['release_date'], album_data.get('album_type', 'album') @@ -3193,27 +3297,40 @@ class MusicDatabase: logger.error(f"Error caching discovery recent album: {e}") return False - def get_discovery_recent_albums(self, limit: int = 10) -> List[Dict[str, Any]]: - """Get cached recent albums for discover page""" + def get_discovery_recent_albums(self, limit: int = 10, source: Optional[str] = None) -> List[Dict[str, Any]]: + """Get cached recent albums for discover page, optionally filtered by source""" try: with self._get_connection() as conn: cursor = conn.cursor() - cursor.execute(""" - SELECT * FROM discovery_recent_albums - ORDER BY release_date DESC - LIMIT ? - """, (limit,)) + if source: + cursor.execute(""" + SELECT * FROM discovery_recent_albums + WHERE source = ? + ORDER BY release_date DESC + LIMIT ? + """, (source, limit)) + else: + cursor.execute(""" + SELECT * FROM discovery_recent_albums + ORDER BY release_date DESC + LIMIT ? + """, (limit,)) rows = cursor.fetchall() + row_keys = rows[0].keys() if rows else [] + return [{ 'album_spotify_id': row['album_spotify_id'], + 'album_itunes_id': row['album_itunes_id'] if 'album_itunes_id' in row_keys else None, 'album_name': row['album_name'], 'artist_name': row['artist_name'], 'artist_spotify_id': row['artist_spotify_id'], + 'artist_itunes_id': row['artist_itunes_id'] if 'artist_itunes_id' in row_keys else None, 'album_cover_url': row['album_cover_url'], 'release_date': row['release_date'], - 'album_type': row['album_type'] + 'album_type': row['album_type'], + 'source': row['source'] if 'source' in row_keys else 'spotify' } for row in rows] except Exception as e: diff --git a/web_server.py b/web_server.py index 753a598d..5995ed27 100644 --- a/web_server.py +++ b/web_server.py @@ -5046,19 +5046,73 @@ def get_artist_image(artist_id): def get_artist_discography(artist_id): """Get an artist's complete discography (albums and singles)""" try: - if not spotify_client or not spotify_client.is_authenticated(): - return jsonify({"error": "Spotify not authenticated"}), 401 - - print(f"🎀 Fetching discography for artist: {artist_id}") - - # Get artist's albums and singles (temporarily include appears_on for debugging) - albums = spotify_client.get_artist_albums(artist_id, album_type='album,single', limit=50) - print(f"πŸ“Š Raw albums returned from Spotify: {len(albums)}") + # Get optional artist name for fallback searches + artist_name = request.args.get('artist_name', '') + + # Determine which source to use + spotify_available = spotify_client and spotify_client.is_spotify_authenticated() + + # Import iTunes client for fallback + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() + + print(f"🎀 Fetching discography for artist: {artist_id} (name: {artist_name}, spotify: {spotify_available})") + + albums = [] + active_source = None + + # Try to get albums from the appropriate source + # Check if the ID looks like Spotify (alphanumeric) or iTunes (numeric only) + is_numeric_id = artist_id.isdigit() + + if spotify_available and not is_numeric_id: + # Try Spotify first for alphanumeric IDs + try: + albums = spotify_client.get_artist_albums(artist_id, album_type='album,single', limit=50) + if albums: + active_source = 'spotify' + print(f"πŸ“Š Got {len(albums)} albums from Spotify") + except Exception as e: + print(f"Spotify lookup failed: {e}") + + # Try iTunes if Spotify didn't work or if it's a numeric ID + if not albums: + try: + if is_numeric_id: + # It's an iTunes ID, use directly + albums = itunes_client.get_artist_albums(artist_id, album_type='album,single', limit=50) + if albums: + active_source = 'itunes' + print(f"πŸ“Š Got {len(albums)} albums from iTunes (direct ID)") + elif artist_name: + # Search iTunes by name + print(f"πŸ”„ Trying iTunes search by name: '{artist_name}'") + itunes_artists = itunes_client.search_artists(artist_name, limit=5) + if itunes_artists: + # Find best match + best_match = None + for artist in itunes_artists: + if artist.name.lower() == artist_name.lower(): + best_match = artist + break + if not best_match: + best_match = itunes_artists[0] + + print(f"βœ… Found iTunes artist: {best_match.name} (ID: {best_match.id})") + albums = itunes_client.get_artist_albums(best_match.id, album_type='album,single', limit=50) + if albums: + active_source = 'itunes' + print(f"πŸ“Š Got {len(albums)} albums from iTunes (name search)") + except Exception as e: + print(f"iTunes lookup failed: {e}") + + print(f"πŸ“Š Total albums returned: {len(albums)} (source: {active_source})") if not albums: return jsonify({ "albums": [], - "singles": [] + "singles": [], + "source": active_source or "unknown" }) # Separate albums from singles/EPs @@ -5073,23 +5127,18 @@ def get_artist_discography(artist_id): if album.id in seen_albums: continue seen_albums.add(album.id) - - # Debug: Check artist information - print(f"πŸ” Checking album: {album.name}") + + # Handle artist matching for both Spotify (objects) and iTunes (strings) formats if hasattr(album, 'artists') and album.artists: - primary_artist_id = album.artists[0].id if hasattr(album.artists[0], 'id') else None - primary_artist_name = album.artists[0].name if hasattr(album.artists[0], 'name') else None - print(f" Primary artist: {primary_artist_name} (ID: {primary_artist_id})") - print(f" Requested artist ID: {artist_id}") - - # Skip if the primary artist doesn't match our requested artist - if primary_artist_id and primary_artist_id != artist_id: - print(f"🚫 Skipping '{album.name}' - primary artist mismatch") - continue - elif not primary_artist_id: - print(f"⚠️ No primary artist ID found for '{album.name}' - including anyway") - else: - print(f"⚠️ No artists found for '{album.name}' - including anyway") + first_artist = album.artists[0] + # Check if artists are objects (Spotify) or strings (iTunes) + if hasattr(first_artist, 'id'): + # Spotify format: artist is an object with .id and .name + primary_artist_id = first_artist.id + # Skip if the primary artist doesn't match our requested artist + if primary_artist_id and primary_artist_id != artist_id: + continue + # iTunes format: artist is a string, can't verify ID match so include all album_data = { "id": album.id, @@ -5138,9 +5187,10 @@ def get_artist_discography(artist_id): return jsonify({ "albums": album_list, - "singles": singles_list + "singles": singles_list, + "source": active_source or "spotify" }) - + except Exception as e: print(f"❌ Error fetching artist discography: {e}") import traceback @@ -18178,84 +18228,141 @@ metadata_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix= # == DISCOVER PAGE ENDPOINTS == # =============================== +def _get_active_discovery_source(): + """ + Determine which music source is active for discovery. + Returns 'spotify' if Spotify is authenticated, 'itunes' otherwise. + """ + if spotify_client and spotify_client.is_spotify_authenticated(): + return 'spotify' + return 'itunes' + + @app.route('/api/discover/hero', methods=['GET']) def get_discover_hero(): """Get featured similar artists for hero slideshow""" try: database = get_database() + # Determine active source + active_source = _get_active_discovery_source() + print(f"🎡 Discover hero using source: {active_source}") + # Get top similar artists (by occurrence count) - get 20 for variety similar_artists = database.get_top_similar_artists(limit=20) if not similar_artists: - return jsonify({"success": True, "artists": []}) + return jsonify({"success": True, "artists": [], "source": active_source}) + + # Filter to artists that have the appropriate ID for the active source + valid_artists = [] + for artist in similar_artists: + if active_source == 'spotify' and artist.similar_artist_spotify_id: + valid_artists.append(artist) + elif active_source == 'itunes' and artist.similar_artist_itunes_id: + valid_artists.append(artist) + # If we have both IDs, include regardless of source + elif artist.similar_artist_spotify_id and artist.similar_artist_itunes_id: + valid_artists.append(artist) # Shuffle for variety and take top 10 import random - shuffled = list(similar_artists) + shuffled = list(valid_artists) random.shuffle(shuffled) similar_artists = shuffled[:10] - # Convert to JSON format with Spotify data enrichment + # Import iTunes client for fallback + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() + + # Convert to JSON format with data enrichment from appropriate source hero_artists = [] for artist in similar_artists: + # Use the ID for the active source, falling back to the other if needed + if active_source == 'spotify': + artist_id = artist.similar_artist_spotify_id or artist.similar_artist_itunes_id + else: + artist_id = artist.similar_artist_itunes_id or artist.similar_artist_spotify_id + artist_data = { "spotify_artist_id": artist.similar_artist_spotify_id, + "itunes_artist_id": artist.similar_artist_itunes_id, + "artist_id": artist_id, # The ID for the current active source "artist_name": artist.similar_artist_name, "occurrence_count": artist.occurrence_count, - "similarity_rank": artist.similarity_rank + "similarity_rank": artist.similarity_rank, + "source": active_source } - # Try to get artist image from Spotify + # Try to get artist image from the active source try: - if spotify_client and spotify_client.is_authenticated(): - sp_artist = spotify_client.get_artist(artist.similar_artist_spotify_id) - if sp_artist and sp_artist.get('images'): - artist_data['image_url'] = sp_artist['images'][0]['url'] if sp_artist['images'] else None - artist_data['genres'] = sp_artist.get('genres', []) - artist_data['popularity'] = sp_artist.get('popularity', 0) - except: - pass + if active_source == 'spotify' and artist.similar_artist_spotify_id: + if spotify_client and spotify_client.is_authenticated(): + sp_artist = spotify_client.get_artist(artist.similar_artist_spotify_id) + if sp_artist and sp_artist.get('images'): + artist_data['image_url'] = sp_artist['images'][0]['url'] if sp_artist['images'] else None + artist_data['genres'] = sp_artist.get('genres', []) + artist_data['popularity'] = sp_artist.get('popularity', 0) + elif active_source == 'itunes' and artist.similar_artist_itunes_id: + itunes_artist = itunes_client.get_artist(artist.similar_artist_itunes_id) + if itunes_artist: + artist_data['image_url'] = itunes_artist.get('images', [{}])[0].get('url') if itunes_artist.get('images') else None + artist_data['genres'] = itunes_artist.get('genres', []) + artist_data['popularity'] = itunes_artist.get('popularity', 0) + except Exception as img_err: + print(f"Could not fetch artist image: {img_err}") hero_artists.append(artist_data) - return jsonify({"success": True, "artists": hero_artists}) + return jsonify({"success": True, "artists": hero_artists, "source": active_source}) except Exception as e: print(f"Error getting discover hero: {e}") return jsonify({"success": False, "error": str(e)}), 500 + @app.route('/api/discover/recent-releases', methods=['GET']) def get_discover_recent_releases(): """Get cached recent albums from watchlist and similar artists""" try: database = get_database() - # Get cached recent albums (max 20) - albums = database.get_discovery_recent_albums(limit=20) + # Determine active source + active_source = _get_active_discovery_source() - return jsonify({"success": True, "albums": albums}) + # Get cached recent albums filtered by source (max 20) + albums = database.get_discovery_recent_albums(limit=20, source=active_source) + + return jsonify({"success": True, "albums": albums, "source": active_source}) except Exception as e: print(f"Error getting recent releases: {e}") return jsonify({"success": False, "error": str(e)}), 500 + @app.route('/api/discover/release-radar', methods=['GET']) def get_discover_release_radar(): """Get release radar playlist - curated selection that stays consistent until next update""" try: database = get_database() - if not spotify_client or not spotify_client.is_authenticated(): - return jsonify({"success": True, "tracks": []}) + # Determine active source - release radar works with any source now + active_source = _get_active_discovery_source() # Try to get curated playlist first curated_track_ids = database.get_curated_playlist('release_radar') if curated_track_ids: - # Use curated selection - fetch track data from discovery pool (same as Discovery Weekly) - discovery_tracks = database.get_discovery_pool_tracks(limit=5000, new_releases_only=False) - tracks_by_id = {track.spotify_track_id: track for track in discovery_tracks} + # Use curated selection - fetch track data from discovery pool filtered by source + discovery_tracks = database.get_discovery_pool_tracks(limit=5000, new_releases_only=False, source=active_source) + + # Build lookup dict with source-appropriate IDs + tracks_by_id = {} + for track in discovery_tracks: + if active_source == 'spotify' and track.spotify_track_id: + tracks_by_id[track.spotify_track_id] = track + elif active_source == 'itunes' and track.itunes_track_id: + tracks_by_id[track.itunes_track_id] = track selected_tracks = [] for track_id in curated_track_ids: @@ -18271,19 +18378,22 @@ def get_discover_release_radar(): track_data = None selected_tracks.append({ + "track_id": track.spotify_track_id or track.itunes_track_id, "spotify_track_id": track.spotify_track_id, + "itunes_track_id": track.itunes_track_id, "track_name": track.track_name, "artist_name": track.artist_name, "album_name": track.album_name, "album_cover_url": track.album_cover_url, "duration_ms": track.duration_ms, - "track_data_json": track_data # Now properly parsed + "track_data_json": track_data, + "source": track.source }) - return jsonify({"success": True, "tracks": selected_tracks}) + return jsonify({"success": True, "tracks": selected_tracks, "source": active_source}) # Fallback: no curated playlist exists (shouldn't happen after first scan) - return jsonify({"success": True, "tracks": []}) + return jsonify({"success": True, "tracks": [], "source": active_source}) except Exception as e: print(f"Error getting release radar: {e}") @@ -18297,13 +18407,23 @@ def get_discover_weekly(): try: database = get_database() + # Determine active source + active_source = _get_active_discovery_source() + # Try to get curated playlist first curated_track_ids = database.get_curated_playlist('discovery_weekly') if curated_track_ids: - # Use curated selection - fetch track data from discovery pool - discovery_tracks = database.get_discovery_pool_tracks(limit=5000, new_releases_only=False) - tracks_by_id = {track.spotify_track_id: track for track in discovery_tracks} + # Use curated selection - fetch track data from discovery pool filtered by source + discovery_tracks = database.get_discovery_pool_tracks(limit=5000, new_releases_only=False, source=active_source) + + # Build lookup dict with source-appropriate IDs + tracks_by_id = {} + for track in discovery_tracks: + if active_source == 'spotify' and track.spotify_track_id: + tracks_by_id[track.spotify_track_id] = track + elif active_source == 'itunes' and track.itunes_track_id: + tracks_by_id[track.itunes_track_id] = track selected_tracks = [] for track_id in curated_track_ids: @@ -18319,19 +18439,22 @@ def get_discover_weekly(): track_data = None selected_tracks.append({ + "track_id": track.spotify_track_id or track.itunes_track_id, "spotify_track_id": track.spotify_track_id, + "itunes_track_id": track.itunes_track_id, "track_name": track.track_name, "artist_name": track.artist_name, "album_name": track.album_name, "album_cover_url": track.album_cover_url, "duration_ms": track.duration_ms, - "track_data_json": track_data # Now properly parsed + "track_data_json": track_data, + "source": track.source }) - return jsonify({"success": True, "tracks": selected_tracks}) + return jsonify({"success": True, "tracks": selected_tracks, "source": active_source}) # Fallback: no curated playlist exists (shouldn't happen after first scan) - return jsonify({"success": True, "tracks": []}) + return jsonify({"success": True, "tracks": [], "source": active_source}) except Exception as e: print(f"Error getting discovery weekly: {e}") diff --git a/webui/static/script.js b/webui/static/script.js index 5aa92be5..88969d05 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -20000,15 +20000,17 @@ async function selectArtistForDetail(artist) { // Update artist info in header updateArtistDetailHeader(artist); - // Load discography - await loadArtistDiscography(artist.id); + // Load discography (pass artist name for cross-source fallback) + await loadArtistDiscography(artist.id, artist.name); } /** - * Load artist's discography from Spotify + * Load artist's discography from Spotify or iTunes + * @param {string} artistId - Artist ID (Spotify or iTunes format) + * @param {string} [artistName] - Optional artist name for fallback searches */ -async function loadArtistDiscography(artistId) { - console.log(`πŸ’Ώ Loading discography for artist: ${artistId}`); +async function loadArtistDiscography(artistId, artistName = null) { + console.log(`πŸ’Ώ Loading discography for artist: ${artistId} (name: ${artistName})`); // Check cache first if (artistsPageState.cache.discography[artistId]) { @@ -20030,8 +20032,14 @@ async function loadArtistDiscography(artistId) { // Show loading states showDiscographyLoading(); + // Build URL with optional artist name for fallback + let url = `/api/artist/${artistId}/discography`; + if (artistName) { + url += `?artist_name=${encodeURIComponent(artistName)}`; + } + // Call the real API endpoint - const response = await fetch(`/api/artist/${artistId}/discography`); + const response = await fetch(url); if (!response.ok) { if (response.status === 401) { From 1d14a8b987efa7d3fb9114b6c5255a3bc5db3563 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Fri, 23 Jan 2026 23:05:58 -0800 Subject: [PATCH 07/20] Discover page itunes integration. Spotify and Itunes will have their own pool --- core/itunes_client.py | 21 ++++- core/metadata_service.py | 34 ++++---- core/personalized_playlists.py | 16 ++-- core/spotify_client.py | 63 ++++++++++---- core/watchlist_scanner.py | 151 +++++++++++++++++++++++++++++++-- database/music_database.py | 151 +++++++++++++++++++++++++++------ web_server.py | 38 +++------ 7 files changed, 377 insertions(+), 97 deletions(-) diff --git a/core/itunes_client.py b/core/itunes_client.py index cf974687..0a5f3d23 100644 --- a/core/itunes_client.py +++ b/core/itunes_client.py @@ -371,8 +371,13 @@ class iTunesClient: return albums[:limit] - def get_album(self, album_id: str) -> Optional[Dict[str, Any]]: - """Get album information - normalized to Spotify format""" + def get_album(self, album_id: str, include_tracks: bool = True) -> Optional[Dict[str, Any]]: + """Get album information with tracks - normalized to Spotify format. + + Args: + album_id: iTunes album/collection ID + include_tracks: If True, also fetches and includes tracks (default True for Spotify compatibility) + """ results = self._lookup(id=album_id) for album_data in results: @@ -400,7 +405,7 @@ class iTunesClient: else: album_type = 'album' - return { + album_result = { 'id': str(album_data.get('collectionId', '')), 'name': album_data.get('collectionName', ''), 'images': images, @@ -414,6 +419,16 @@ class iTunesClient: '_raw_data': album_data } + # Include tracks to match Spotify's get_album format + if include_tracks: + tracks_data = self.get_album_tracks(album_id) + if tracks_data and 'items' in tracks_data: + album_result['tracks'] = tracks_data + else: + album_result['tracks'] = {'items': [], 'total': 0} + + return album_result + return None def get_album_tracks(self, album_id: str) -> Optional[Dict[str, Any]]: diff --git a/core/metadata_service.py b/core/metadata_service.py index c4225c07..e1b6fe70 100644 --- a/core/metadata_service.py +++ b/core/metadata_service.py @@ -43,7 +43,7 @@ class MetadataService: def _log_initialization(self): """Log initialization status""" - spotify_status = "βœ… Authenticated" if self.spotify.is_authenticated() else "❌ Not authenticated" + spotify_status = "βœ… Authenticated" if self.spotify.is_spotify_authenticated() else "❌ Not authenticated" itunes_status = "βœ… Available" if self.itunes.is_authenticated() else "❌ Not available" logger.info(f"MetadataService initialized - Spotify: {spotify_status}, iTunes: {itunes_status}") @@ -52,7 +52,7 @@ class MetadataService: def get_active_provider(self) -> str: """ Get the currently active metadata provider. - + Returns: "spotify" or "itunes" """ @@ -61,14 +61,16 @@ class MetadataService: elif self.preferred_provider == "itunes": return "itunes" else: # auto - return "spotify" if self.spotify.is_authenticated() else "itunes" + # Use is_spotify_authenticated() to check actual Spotify auth status + # (is_authenticated() always returns True due to iTunes fallback) + return "spotify" if self.spotify.is_spotify_authenticated() else "itunes" def _get_client(self): """Get the appropriate client based on provider selection""" provider = self.get_active_provider() - + if provider == "spotify": - if not self.spotify.is_authenticated(): + if not self.spotify.is_spotify_authenticated(): logger.warning("Spotify requested but not authenticated, falling back to iTunes") return self.itunes return self.spotify @@ -168,38 +170,38 @@ class MetadataService: def get_user_playlists(self) -> List: """Get user playlists (Spotify only)""" - if self.get_active_provider() == "spotify" and self.spotify.is_authenticated(): + if self.spotify.is_spotify_authenticated(): return self.spotify.get_user_playlists() logger.warning("User playlists only available with Spotify authentication") return [] - + def get_saved_tracks(self) -> List: """Get user's saved/liked tracks (Spotify only)""" - if self.get_active_provider() == "spotify" and self.spotify.is_authenticated(): + if self.spotify.is_spotify_authenticated(): return self.spotify.get_saved_tracks() logger.warning("Saved tracks only available with Spotify authentication") return [] - + def get_saved_tracks_count(self) -> int: """Get count of user's saved tracks (Spotify only)""" - if self.get_active_provider() == "spotify" and self.spotify.is_authenticated(): + if self.spotify.is_spotify_authenticated(): return self.spotify.get_saved_tracks_count() return 0 - + # ==================== Utility Methods ==================== - + def is_authenticated(self) -> bool: """Check if any provider is available""" - return self.spotify.is_authenticated() or self.itunes.is_authenticated() - + return self.spotify.is_spotify_authenticated() or self.itunes.is_authenticated() + def get_provider_info(self) -> Dict[str, Any]: """Get information about available providers""" return { "active_provider": self.get_active_provider(), - "spotify_authenticated": self.spotify.is_authenticated(), + "spotify_authenticated": self.spotify.is_spotify_authenticated(), "itunes_available": self.itunes.is_authenticated(), "preferred_provider": self.preferred_provider, - "can_access_user_data": self.spotify.is_authenticated(), + "can_access_user_data": self.spotify.is_spotify_authenticated(), } def reload_config(self): diff --git a/core/personalized_playlists.py b/core/personalized_playlists.py index 557ee394..7a7fa94a 100644 --- a/core/personalized_playlists.py +++ b/core/personalized_playlists.py @@ -112,7 +112,11 @@ class PersonalizedPlaylistsService: def _build_track_dict(self, row, source: str) -> Dict: """Build a standardized track dictionary from a database row.""" - track_data = row['track_data_json'] + # Convert sqlite3.Row to dict if needed (Row objects don't support .get()) + if hasattr(row, 'keys'): + row = dict(row) + + track_data = row.get('track_data_json') if isinstance(track_data, str): try: track_data = json.loads(track_data) @@ -123,11 +127,11 @@ class PersonalizedPlaylistsService: 'track_id': row.get('spotify_track_id') or row.get('itunes_track_id'), 'spotify_track_id': row.get('spotify_track_id'), 'itunes_track_id': row.get('itunes_track_id'), - 'track_name': row['track_name'], - 'artist_name': row['artist_name'], - 'album_name': row['album_name'], - 'album_cover_url': row['album_cover_url'], - 'duration_ms': row['duration_ms'], + 'track_name': row.get('track_name', 'Unknown'), + 'artist_name': row.get('artist_name', 'Unknown'), + 'album_name': row.get('album_name', 'Unknown'), + 'album_cover_url': row.get('album_cover_url'), + 'duration_ms': row.get('duration_ms', 0), 'popularity': row.get('popularity', 0), 'track_data_json': track_data, 'source': source diff --git a/core/spotify_client.py b/core/spotify_client.py index 304f39de..9279db60 100644 --- a/core/spotify_client.py +++ b/core/spotify_client.py @@ -172,6 +172,19 @@ class SpotifyClient: self._itunes_client = None # Lazy-loaded iTunes fallback self._setup_client() + def _is_spotify_id(self, id_str: str) -> bool: + """Check if an ID is a Spotify ID (alphanumeric) vs iTunes ID (numeric only)""" + if not id_str: + return False + # Spotify IDs contain letters and numbers, iTunes IDs are purely numeric + return not id_str.isdigit() + + def _is_itunes_id(self, id_str: str) -> bool: + """Check if an ID is an iTunes ID (numeric only)""" + if not id_str: + return False + return id_str.isdigit() + @property def _itunes(self): """Lazy-load iTunes client for fallback when Spotify not authenticated""" @@ -534,9 +547,13 @@ class SpotifyClient: logger.error(f"Error fetching track details via Spotify: {e}") # Fall through to iTunes fallback - # iTunes fallback - logger.debug(f"Using iTunes fallback for track details: {track_id}") - return self._itunes.get_track_details(track_id) + # iTunes fallback - only if ID is numeric (iTunes format) + if self._is_itunes_id(track_id): + logger.debug(f"Using iTunes fallback for track details: {track_id}") + return self._itunes.get_track_details(track_id) + else: + logger.debug(f"Cannot use iTunes fallback for Spotify track ID: {track_id}") + return None @rate_limited def get_track_features(self, track_id: str) -> Optional[Dict[str, Any]]: @@ -563,9 +580,13 @@ class SpotifyClient: logger.error(f"Error fetching album via Spotify: {e}") # Fall through to iTunes fallback - # iTunes fallback - logger.debug(f"Using iTunes fallback for album: {album_id}") - return self._itunes.get_album(album_id) + # iTunes fallback - only if ID is numeric (iTunes format) + if self._is_itunes_id(album_id): + logger.debug(f"Using iTunes fallback for album: {album_id}") + return self._itunes.get_album(album_id) + else: + logger.debug(f"Cannot use iTunes fallback for Spotify album ID: {album_id}") + return None @rate_limited def get_album_tracks(self, album_id: str) -> Optional[Dict[str, Any]]: @@ -602,9 +623,13 @@ class SpotifyClient: logger.error(f"Error fetching album tracks via Spotify: {e}") # Fall through to iTunes fallback - # iTunes fallback - logger.debug(f"Using iTunes fallback for album tracks: {album_id}") - return self._itunes.get_album_tracks(album_id) + # iTunes fallback - only if ID is numeric (iTunes format) + if self._is_itunes_id(album_id): + logger.debug(f"Using iTunes fallback for album tracks: {album_id}") + return self._itunes.get_album_tracks(album_id) + else: + logger.debug(f"Cannot use iTunes fallback for Spotify album ID: {album_id}") + return None @rate_limited def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 50) -> List[Album]: @@ -629,9 +654,13 @@ class SpotifyClient: logger.error(f"Error fetching artist albums via Spotify: {e}") # Fall through to iTunes fallback - # iTunes fallback - logger.debug(f"Using iTunes fallback for artist albums: {artist_id}") - return self._itunes.get_artist_albums(artist_id, album_type, limit) + # iTunes fallback - only if ID is numeric (iTunes format) + if self._is_itunes_id(artist_id): + logger.debug(f"Using iTunes fallback for artist albums: {artist_id}") + return self._itunes.get_artist_albums(artist_id, album_type, limit) + else: + logger.debug(f"Cannot use iTunes fallback for Spotify artist ID: {artist_id}") + return [] @rate_limited def get_user_info(self) -> Optional[Dict[str, Any]]: @@ -662,6 +691,10 @@ class SpotifyClient: logger.error(f"Error fetching artist via Spotify: {e}") # Fall through to iTunes fallback - # iTunes fallback - logger.debug(f"Using iTunes fallback for artist: {artist_id}") - return self._itunes.get_artist(artist_id) \ No newline at end of file + # iTunes fallback - only if ID is numeric (iTunes format) + if self._is_itunes_id(artist_id): + logger.debug(f"Using iTunes fallback for artist: {artist_id}") + return self._itunes.get_artist(artist_id) + else: + logger.debug(f"Cannot use iTunes fallback for Spotify artist ID: {artist_id}") + return None \ No newline at end of file diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 35b65bb6..e331f834 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -280,6 +280,7 @@ class WatchlistScanner: def _get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist): """ Get the appropriate client and artist ID based on active provider. + If iTunes ID is missing, searches by artist name to find and cache it. Returns: Tuple of (client, artist_id, provider_name) or (None, None, None) if no valid ID @@ -296,9 +297,80 @@ class WatchlistScanner: if watchlist_artist.itunes_artist_id: return (self.metadata_service.itunes, watchlist_artist.itunes_artist_id, 'itunes') else: - logger.warning(f"No iTunes ID for {watchlist_artist.artist_name}, cannot scan with iTunes") - return (None, None, None) - + # No iTunes ID stored - search by name and cache it + logger.info(f"No iTunes ID for {watchlist_artist.artist_name}, searching by name...") + try: + itunes_client = self.metadata_service.itunes + search_results = itunes_client.search_artists(watchlist_artist.artist_name, limit=1) + if search_results and len(search_results) > 0: + itunes_id = search_results[0].id + logger.info(f"Found iTunes ID {itunes_id} for {watchlist_artist.artist_name}") + # Cache the iTunes ID in the database for future use + self.database.update_watchlist_artist_itunes_id( + watchlist_artist.spotify_artist_id or str(watchlist_artist.id), + itunes_id + ) + return (itunes_client, itunes_id, 'itunes') + else: + logger.warning(f"Could not find {watchlist_artist.artist_name} on iTunes") + return (None, None, None) + except Exception as e: + logger.error(f"Error searching iTunes for {watchlist_artist.artist_name}: {e}") + return (None, None, None) + + def get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist): + """ + Public wrapper for _get_active_client_and_artist_id. + Gets the appropriate client and artist ID based on active provider. + + Returns: + Tuple of (client, artist_id, provider_name) or (None, None, None) if no valid ID + """ + return self._get_active_client_and_artist_id(watchlist_artist) + + def get_artist_image_url(self, watchlist_artist: WatchlistArtist) -> Optional[str]: + """ + Get artist image URL using the active provider. + + Returns: + Image URL string or None if not available + """ + client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist) + if not client or not artist_id: + return None + + try: + artist_data = client.get_artist(artist_id) + if artist_data: + # Handle both Spotify and iTunes response formats + if 'images' in artist_data and artist_data['images']: + return artist_data['images'][0].get('url') + elif 'image_url' in artist_data: + return artist_data['image_url'] + except Exception as e: + logger.debug(f"Could not fetch artist image for {watchlist_artist.artist_name}: {e}") + + return None + + def get_artist_discography_for_watchlist(self, watchlist_artist: WatchlistArtist, last_scan_timestamp: Optional[datetime] = None) -> Optional[List]: + """ + Get artist's discography using the active provider, with proper ID resolution. + This is the provider-aware version of get_artist_discography. + + Args: + watchlist_artist: WatchlistArtist object (has both spotify and itunes IDs) + last_scan_timestamp: Only return releases after this date (for incremental scans) + + Returns: + List of albums or None on error + """ + client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist) + if not client or not artist_id: + logger.warning(f"No valid client/ID for {watchlist_artist.artist_name}") + return None + + return self._get_artist_discography_with_client(client, artist_id, last_scan_timestamp) + def scan_all_watchlist_artists(self) -> List[ScanResult]: """ Scan artists in the watchlist for new releases. @@ -562,7 +634,9 @@ class WatchlistScanner: try: # Check if we have fresh similar artists cached (< 30 days old) if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30): - logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping fetch") + logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping MusicMap fetch") + # Even if cached, backfill missing iTunes IDs (seamless dual-source support) + self._backfill_similar_artists_itunes_ids(source_artist_id) else: logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}...") self.update_similar_artists(watchlist_artist) @@ -812,6 +886,10 @@ class WatchlistScanner: album_date = datetime(int(year), int(month), 1, tzinfo=timezone.utc) elif len(release_date_str) == 10: # Full date (e.g., "2023-10-15") album_date = datetime.strptime(release_date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc) + elif 'T' in release_date_str: # ISO 8601 with time (e.g., "2017-12-08T08:00:00Z" from iTunes) + # Strip the time portion and parse just the date + date_part = release_date_str.split('T')[0] + album_date = datetime.strptime(date_part, "%Y-%m-%d").replace(tzinfo=timezone.utc) else: logger.warning(f"Unknown release date format: {release_date_str}") return True # Include if we can't parse @@ -1246,6 +1324,54 @@ class WatchlistScanner: logger.error(f"Error fetching similar artists from MusicMap: {e}") return [] + def _backfill_similar_artists_itunes_ids(self, source_artist_id: str) -> int: + """ + Backfill missing iTunes IDs for cached similar artists. + This ensures seamless dual-source support without clearing cached data. + + Args: + source_artist_id: The source artist ID to backfill similar artists for + + Returns: + Number of similar artists updated with iTunes IDs + """ + try: + # Get similar artists that are missing iTunes IDs + similar_artists = self.database.get_similar_artists_missing_itunes_ids(source_artist_id) + + if not similar_artists: + return 0 + + logger.info(f"Backfilling iTunes IDs for {len(similar_artists)} similar artists") + + # Get iTunes client + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() + + updated_count = 0 + for similar_artist in similar_artists: + try: + # Search iTunes by artist name + itunes_results = itunes_client.search_artists(similar_artist.similar_artist_name, limit=1) + if itunes_results and len(itunes_results) > 0: + itunes_id = itunes_results[0].id + # Update the similar artist with the iTunes ID + if self.database.update_similar_artist_itunes_id(similar_artist.id, itunes_id): + updated_count += 1 + logger.debug(f" Backfilled iTunes ID {itunes_id} for {similar_artist.similar_artist_name}") + except Exception as e: + logger.debug(f" Could not backfill iTunes ID for {similar_artist.similar_artist_name}: {e}") + continue + + if updated_count > 0: + logger.info(f"Backfilled {updated_count} similar artists with iTunes IDs") + + return updated_count + + except Exception as e: + logger.error(f"Error backfilling similar artists iTunes IDs: {e}") + return 0 + def update_similar_artists(self, watchlist_artist: WatchlistArtist, limit: int = 10) -> bool: """ Fetch and store similar artists for a watchlist artist. @@ -1351,8 +1477,21 @@ class WatchlistScanner: sources_to_process = [] # Always add iTunes first (baseline source) - if similar_artist.similar_artist_itunes_id: - sources_to_process.append(('itunes', similar_artist.similar_artist_itunes_id)) + itunes_id = similar_artist.similar_artist_itunes_id + if not itunes_id: + # On-the-fly lookup for missing iTunes ID (seamless provider switching) + try: + itunes_results = itunes_client.search_artists(similar_artist.similar_artist_name, limit=1) + if itunes_results and len(itunes_results) > 0: + itunes_id = itunes_results[0].id + # Cache it for future use + self.database.update_similar_artist_itunes_id(similar_artist.id, itunes_id) + logger.debug(f" Resolved iTunes ID {itunes_id} for {similar_artist.similar_artist_name}") + except Exception as e: + logger.debug(f" Could not resolve iTunes ID for {similar_artist.similar_artist_name}: {e}") + + if itunes_id: + sources_to_process.append(('itunes', itunes_id)) # Add Spotify if authenticated and we have an ID if spotify_available and similar_artist.similar_artist_spotify_id: diff --git a/database/music_database.py b/database/music_database.py index 9d198219..9a08b81b 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -591,29 +591,7 @@ class MusicDatabase: ) """) - # Create indexes for performance - cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_source ON similar_artists (source_artist_id)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_spotify ON similar_artists (similar_artist_spotify_id)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_itunes ON similar_artists (similar_artist_itunes_id)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_occurrence ON similar_artists (occurrence_count)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_name ON similar_artists (similar_artist_name)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_spotify_track ON discovery_pool (spotify_track_id)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_itunes_track ON discovery_pool (itunes_track_id)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_artist ON discovery_pool (spotify_artist_id)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_itunes_artist ON discovery_pool (itunes_artist_id)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_source ON discovery_pool (source)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_added_date ON discovery_pool (added_date)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_is_new ON discovery_pool (is_new_release)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_watchlist ON recent_releases (watchlist_artist_id)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_date ON recent_releases (release_date)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_source ON recent_releases (source)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_source ON discovery_recent_albums (source)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_date ON discovery_recent_albums (release_date)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_playlists_type ON listenbrainz_playlists (playlist_type)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_playlists_mbid ON listenbrainz_playlists (playlist_mbid)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_tracks_playlist ON listenbrainz_tracks (playlist_id)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_tracks_position ON listenbrainz_tracks (playlist_id, position)") - cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_artist ON discovery_recent_albums (artist_spotify_id)") + # ============== MIGRATIONS (must run BEFORE index creation on new columns) ============== # Add genres column to discovery_pool if it doesn't exist (migration) cursor.execute("PRAGMA table_info(discovery_pool)") @@ -658,6 +636,30 @@ class MusicDatabase: cursor.execute("ALTER TABLE discovery_recent_albums ADD COLUMN source TEXT DEFAULT 'spotify'") logger.info("Added iTunes columns to discovery_recent_albums table for dual-source discovery") + # ============== INDEXES (after migrations to ensure columns exist) ============== + cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_source ON similar_artists (source_artist_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_spotify ON similar_artists (similar_artist_spotify_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_itunes ON similar_artists (similar_artist_itunes_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_occurrence ON similar_artists (occurrence_count)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_name ON similar_artists (similar_artist_name)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_spotify_track ON discovery_pool (spotify_track_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_itunes_track ON discovery_pool (itunes_track_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_artist ON discovery_pool (spotify_artist_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_itunes_artist ON discovery_pool (itunes_artist_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_source ON discovery_pool (source)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_added_date ON discovery_pool (added_date)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_is_new ON discovery_pool (is_new_release)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_watchlist ON recent_releases (watchlist_artist_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_date ON recent_releases (release_date)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_source ON recent_releases (source)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_source ON discovery_recent_albums (source)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_date ON discovery_recent_albums (release_date)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_playlists_type ON listenbrainz_playlists (playlist_type)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_playlists_mbid ON listenbrainz_playlists (playlist_mbid)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_tracks_playlist ON listenbrainz_tracks (playlist_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_tracks_position ON listenbrainz_tracks (playlist_id, position)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_artist ON discovery_recent_albums (artist_spotify_id)") + logger.info("Discovery tables created successfully") except Exception as e: @@ -2996,6 +2998,27 @@ class MusicDatabase: logger.error(f"Error updating watchlist iTunes ID: {e}") return False + def update_watchlist_artist_itunes_id(self, spotify_artist_id: str, itunes_id: str) -> bool: + """Update the iTunes artist ID for a watchlist artist by Spotify ID (for cross-provider caching)""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute(""" + UPDATE watchlist_artists + SET itunes_artist_id = ?, updated_at = CURRENT_TIMESTAMP + WHERE spotify_artist_id = ? + """, (itunes_id, spotify_artist_id)) + + conn.commit() + if cursor.rowcount > 0: + logger.info(f"Cached iTunes ID {itunes_id} for Spotify artist {spotify_artist_id}") + return cursor.rowcount > 0 + + except Exception as e: + logger.error(f"Error caching watchlist iTunes ID: {e}") + return False + # === Discovery Feature Methods === def add_or_update_similar_artist(self, source_artist_id: str, similar_artist_name: str, @@ -3056,10 +3079,66 @@ class MusicDatabase: logger.error(f"Error getting similar artists: {e}") return [] - def has_fresh_similar_artists(self, source_artist_id: str, days_threshold: int = 30) -> bool: + def get_similar_artists_missing_itunes_ids(self, source_artist_id: str) -> List[SimilarArtist]: + """Get similar artists for a source that are missing iTunes IDs (for backfill)""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute(""" + SELECT * FROM similar_artists + WHERE source_artist_id = ? + AND (similar_artist_itunes_id IS NULL OR similar_artist_itunes_id = '') + ORDER BY occurrence_count DESC + LIMIT 50 + """, (source_artist_id,)) + + rows = cursor.fetchall() + return [SimilarArtist( + id=row['id'], + source_artist_id=row['source_artist_id'], + similar_artist_spotify_id=row['similar_artist_spotify_id'], + similar_artist_itunes_id=None, + similar_artist_name=row['similar_artist_name'], + similarity_rank=row['similarity_rank'], + occurrence_count=row['occurrence_count'], + last_updated=datetime.fromisoformat(row['last_updated']) + ) for row in rows] + + except Exception as e: + logger.error(f"Error getting similar artists missing iTunes IDs: {e}") + return [] + + def update_similar_artist_itunes_id(self, similar_artist_id: int, itunes_id: str) -> bool: + """Update a similar artist's iTunes ID (for backfill)""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute(""" + UPDATE similar_artists + SET similar_artist_itunes_id = ? + WHERE id = ? + """, (itunes_id, similar_artist_id)) + + conn.commit() + return cursor.rowcount > 0 + + except Exception as e: + logger.error(f"Error updating similar artist iTunes ID: {e}") + return False + + def has_fresh_similar_artists(self, source_artist_id: str, days_threshold: int = 30, require_itunes: bool = True) -> bool: """ Check if we have cached similar artists that are still fresh (< days_threshold old). - Returns True if we have recent data, False if data is stale or missing. + Also checks that similar artists have the required provider IDs. + + Args: + source_artist_id: The source artist ID to check + days_threshold: Maximum age in days to consider fresh + require_itunes: If True, also requires iTunes IDs to be present (for seamless provider switching) + + Returns True if we have recent data with required IDs, False if data is stale, missing, or incomplete. """ try: with self._get_connection() as conn: @@ -3081,7 +3160,27 @@ class MusicDatabase: last_updated = datetime.fromisoformat(row['last_updated']) days_since_update = (datetime.now() - last_updated).total_seconds() / 86400 # seconds to days - return days_since_update < days_threshold + if days_since_update >= days_threshold: + return False + + # Check if we have iTunes IDs (for seamless provider switching) + if require_itunes: + cursor.execute(""" + SELECT COUNT(*) as total, + SUM(CASE WHEN similar_artist_itunes_id IS NOT NULL AND similar_artist_itunes_id != '' THEN 1 ELSE 0 END) as has_itunes + FROM similar_artists + WHERE source_artist_id = ? + """, (source_artist_id,)) + id_row = cursor.fetchone() + + if id_row and id_row['total'] > 0: + # If less than 50% have iTunes IDs, consider stale and refetch + itunes_ratio = id_row['has_itunes'] / id_row['total'] + if itunes_ratio < 0.5: + logger.debug(f"Similar artists for {source_artist_id} missing iTunes IDs ({id_row['has_itunes']}/{id_row['total']}), will refetch") + return False + + return True except Exception as e: logger.error(f"Error checking similar artists freshness: {e}") diff --git a/web_server.py b/web_server.py index 5995ed27..f2826a02 100644 --- a/web_server.py +++ b/web_server.py @@ -17269,14 +17269,8 @@ def start_watchlist_scan(): for i, artist in enumerate(watchlist_artists): try: - # Fetch artist image - artist_image_url = '' - try: - artist_data = spotify_client.get_artist(artist.spotify_artist_id) - if artist_data and 'images' in artist_data and artist_data['images']: - artist_image_url = artist_data['images'][0]['url'] - except: - pass + # Fetch artist image using provider-aware method + artist_image_url = scanner.get_artist_image_url(artist) or '' # Update progress watchlist_scan_state.update({ @@ -17290,9 +17284,9 @@ def start_watchlist_scan(): 'current_album_image_url': '', 'current_track_name': '' }) - - # Get artist discography - albums = scanner.get_artist_discography(artist.spotify_artist_id, artist.last_scan_timestamp) + + # Get artist discography using provider-aware method + albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp) if albums is None: scan_results.append(type('ScanResult', (), { @@ -17320,8 +17314,8 @@ def start_watchlist_scan(): # Scan each album for album_index, album in enumerate(albums): try: - # Get album tracks - album_data = scanner.spotify_client.get_album(album.id) + # Get album tracks using provider-aware method + album_data = scanner.metadata_service.get_album(album.id) if not album_data or 'tracks' not in album_data: continue @@ -17969,14 +17963,8 @@ def _process_watchlist_scan_automatically(): # Scan each artist with detailed tracking for i, artist in enumerate(watchlist_artists): try: - # Fetch artist image - artist_image_url = '' - try: - artist_data = spotify_client.get_artist(artist.spotify_artist_id) - if artist_data and 'images' in artist_data and artist_data['images']: - artist_image_url = artist_data['images'][0]['url'] - except: - pass + # Fetch artist image using provider-aware method + artist_image_url = scanner.get_artist_image_url(artist) or '' # Update progress watchlist_scan_state.update({ @@ -17991,8 +17979,8 @@ def _process_watchlist_scan_automatically(): 'current_track_name': '' }) - # Get artist discography - albums = scanner.get_artist_discography(artist.spotify_artist_id, artist.last_scan_timestamp) + # Get artist discography using provider-aware method + albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp) if albums is None: scan_results.append(type('ScanResult', (), { @@ -18020,8 +18008,8 @@ def _process_watchlist_scan_automatically(): # Scan each album for album_index, album in enumerate(albums): try: - # Get album tracks - album_data = scanner.spotify_client.get_album(album.id) + # Get album tracks using provider-aware method + album_data = scanner.metadata_service.get_album(album.id) if not album_data or 'tracks' not in album_data: continue From 4319d440eed916da4e52e4f1a78ffca40f3e046a Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sat, 24 Jan 2026 09:47:41 -0800 Subject: [PATCH 08/20] Add iTunes as resilient primary source for discovery features Implement dual-source architecture where iTunes serves as always-available primary source and Spotify as preferred source when authenticated. - Make watchlist scans provider-aware (manual and auto paths) - Update discovery pool population to process both sources - Update recent albums caching for both sources - Create source-specific curated playlists (Fresh Tape, Archives) - Add on-the-fly iTunes ID resolution for similar artists - Add iTunes ID check to similar artists freshness validation - Fix sqlite3.Row compatibility in personalized playlists - Fix iTunes ISO 8601 date format parsing - Update API endpoints to serve source-appropriate data This ensures the app remains fully functional if Spotify becomes unavailable (rate limits, auth issues, bans) by seamlessly falling back to iTunes data that has been building in parallel. --- core/watchlist_scanner.py | 498 ++++++++++++++++++++------------------ web_server.py | 12 +- 2 files changed, 273 insertions(+), 237 deletions(-) diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index e331f834..db7c9b17 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -1990,112 +1990,148 @@ class WatchlistScanner: """ Cache recent albums from watchlist and similar artists for discover page. - IMPROVED: Checks ALL watchlist artists + top similar artists with 14-day window - (like Spotify's Release Radar) for more comprehensive and fresh content. + Supports both Spotify and iTunes sources - iTunes is always processed (baseline), + Spotify is added when authenticated. Same pattern as discovery pool. """ try: from datetime import datetime, timedelta - import random logger.info("Caching recent albums for discover page...") # Clear existing cache self.database.clear_discovery_recent_albums() - # IMPROVED: 30-day window for better content variety while staying recent + # 30-day window for recent releases cutoff_date = datetime.now() - timedelta(days=30) - cached_count = 0 + cached_count = {'spotify': 0, 'itunes': 0} albums_checked = 0 - # IMPROVED: Check ALL watchlist artists (not random 10) - watchlist_artists = self.database.get_watchlist_artists() + # Determine available sources + spotify_available = self.spotify_client and self.spotify_client.is_spotify_authenticated() - # IMPROVED: Check top 50 similar artists (not random 10 from 30) + # Get iTunes client + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() + + # Get artists to check + watchlist_artists = self.database.get_watchlist_artists() similar_artists = self.database.get_top_similar_artists(limit=50) - logger.info(f"Checking albums from {len(watchlist_artists)} watchlist + {len(similar_artists)} similar artists for recent releases (last 14 days)") + logger.info(f"Checking albums from {len(watchlist_artists)} watchlist + {len(similar_artists)} similar artists") + logger.info(f"Sources: Spotify={spotify_available}, iTunes=True") + + def process_album(album, artist_name, artist_spotify_id, artist_itunes_id, source): + """Helper to process and cache a single album""" + nonlocal albums_checked + try: + albums_checked += 1 + release_str = album.release_date if hasattr(album, 'release_date') else None + + if not release_str: + return False + + # Handle iTunes ISO format (2017-12-08T08:00:00Z) + if 'T' in release_str: + release_str = release_str.split('T')[0] + + if len(release_str) >= 10: + release_date = datetime.strptime(release_str[:10], "%Y-%m-%d") + if release_date >= cutoff_date: + album_data = { + 'album_spotify_id': album.id if source == 'spotify' else None, + 'album_itunes_id': album.id if source == 'itunes' else None, + 'album_name': album.name, + 'artist_name': artist_name, + 'artist_spotify_id': artist_spotify_id, + 'artist_itunes_id': artist_itunes_id, + 'album_cover_url': album.image_url if hasattr(album, 'image_url') else None, + 'release_date': release_str[:10], + 'album_type': album.album_type if hasattr(album, 'album_type') else 'album' + } + if self.database.cache_discovery_recent_album(album_data, source=source): + cached_count[source] += 1 + logger.debug(f"Cached [{source}] recent album: {album.name} by {artist_name} ({release_str})") + return True + except Exception as e: + logger.debug(f"Error processing album: {e}") + return False # Process watchlist artists for artist in watchlist_artists: - try: - albums = self.spotify_client.get_artist_albums( - artist.spotify_artist_id, - album_type='album,single,ep', # Include EPs for comprehensive coverage - limit=20 - ) + # Always process iTunes (baseline) + itunes_id = artist.itunes_artist_id + if not itunes_id: + # Try to resolve iTunes ID on-the-fly + try: + results = itunes_client.search_artists(artist.artist_name, limit=1) + if results and len(results) > 0: + itunes_id = results[0].id + except: + pass - for album in albums: - try: - albums_checked += 1 - if hasattr(album, 'release_date') and album.release_date: - release_str = album.release_date - if len(release_str) >= 10: - release_date = datetime.strptime(release_str[:10], "%Y-%m-%d") - if release_date >= cutoff_date: - album_data = { - 'album_spotify_id': album.id, - 'album_name': album.name, - 'artist_name': artist.artist_name, - 'artist_spotify_id': artist.spotify_artist_id, - 'album_cover_url': album.image_url if hasattr(album, 'image_url') else None, - 'release_date': release_str, - 'album_type': album.album_type if hasattr(album, 'album_type') else 'album' - } - if self.database.cache_discovery_recent_album(album_data): - cached_count += 1 - logger.debug(f"Cached recent album: {album.name} by {artist.artist_name} ({release_str})") - except Exception as e: - logger.warning(f"Error checking album for recent releases: {e}") - continue + if itunes_id: + try: + albums = itunes_client.get_artist_albums(itunes_id, album_type='album,single', limit=20) + for album in albums or []: + process_album(album, artist.artist_name, artist.spotify_artist_id, itunes_id, 'itunes') + except Exception as e: + logger.debug(f"Error fetching iTunes albums for {artist.artist_name}: {e}") - except Exception as e: - logger.debug(f"Error fetching albums for watchlist artist {artist.artist_name}: {e}") - continue + # Process Spotify if authenticated + if spotify_available and artist.spotify_artist_id: + try: + albums = self.spotify_client.get_artist_albums( + artist.spotify_artist_id, + album_type='album,single,ep', + limit=20 + ) + for album in albums or []: + process_album(album, artist.artist_name, artist.spotify_artist_id, itunes_id, 'spotify') + except Exception as e: + logger.debug(f"Error fetching Spotify albums for {artist.artist_name}: {e}") - # Rate limiting between artists time.sleep(DELAY_BETWEEN_ARTISTS) # Process similar artists for artist in similar_artists: - try: - albums = self.spotify_client.get_artist_albums( - artist.similar_artist_spotify_id, - album_type='album,single,ep', # Include EPs for comprehensive coverage - limit=20 - ) + # Always process iTunes (baseline) + itunes_id = artist.similar_artist_itunes_id + if not itunes_id: + # Try to resolve iTunes ID on-the-fly + try: + results = itunes_client.search_artists(artist.similar_artist_name, limit=1) + if results and len(results) > 0: + itunes_id = results[0].id + # Cache for future + self.database.update_similar_artist_itunes_id(artist.id, itunes_id) + except: + pass - for album in albums: - try: - albums_checked += 1 - if hasattr(album, 'release_date') and album.release_date: - release_str = album.release_date - if len(release_str) >= 10: - release_date = datetime.strptime(release_str[:10], "%Y-%m-%d") - if release_date >= cutoff_date: - album_data = { - 'album_spotify_id': album.id, - 'album_name': album.name, - 'artist_name': artist.similar_artist_name, - 'artist_spotify_id': artist.similar_artist_spotify_id, - 'album_cover_url': album.image_url if hasattr(album, 'image_url') else None, - 'release_date': release_str, - 'album_type': album.album_type if hasattr(album, 'album_type') else 'album' - } - if self.database.cache_discovery_recent_album(album_data): - cached_count += 1 - logger.debug(f"Cached recent album: {album.name} by {artist.similar_artist_name} ({release_str})") - except Exception as e: - logger.warning(f"Error checking album for recent releases: {e}") - continue + if itunes_id: + try: + albums = itunes_client.get_artist_albums(itunes_id, album_type='album,single', limit=20) + for album in albums or []: + process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, itunes_id, 'itunes') + except Exception as e: + logger.debug(f"Error fetching iTunes albums for {artist.similar_artist_name}: {e}") - except Exception as e: - logger.debug(f"Error fetching albums for similar artist {artist.similar_artist_name}: {e}") - continue + # Process Spotify if authenticated + if spotify_available and artist.similar_artist_spotify_id: + try: + albums = self.spotify_client.get_artist_albums( + artist.similar_artist_spotify_id, + album_type='album,single,ep', + limit=20 + ) + for album in albums or []: + process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, itunes_id, 'spotify') + except Exception as e: + logger.debug(f"Error fetching Spotify albums for {artist.similar_artist_name}: {e}") - # Rate limiting between artists time.sleep(DELAY_BETWEEN_ARTISTS) - logger.info(f"Cached {cached_count} recent albums from {albums_checked} albums checked (cutoff: {cutoff_date.strftime('%Y-%m-%d')})") + total_cached = cached_count['spotify'] + cached_count['itunes'] + logger.info(f"Cached {total_cached} recent albums (Spotify: {cached_count['spotify']}, iTunes: {cached_count['itunes']}) from {albums_checked} albums checked") except Exception as e: logger.error(f"Error caching discovery recent albums: {e}") @@ -2106,45 +2142,67 @@ class WatchlistScanner: """ Curate consistent playlist selections that stay the same until next discovery pool update. - IMPROVED: Spotify-quality curation with popularity scoring and smart algorithms + Supports both Spotify and iTunes sources - creates separate curated playlists for each. - Release Radar: Prioritizes freshness + popularity from recent releases - Discovery Weekly: Balanced mix of popular picks, deep cuts, and mid-tier tracks """ try: import random from datetime import datetime + from core.itunes_client import iTunesClient - logger.info("Curating Release Radar playlist...") + logger.info("Curating discovery playlists...") - # 1. Curate Release Radar - 50 tracks from recent albums - # IMPROVED: Get more albums (50 instead of 20) for better selection - recent_albums = self.database.get_discovery_recent_albums(limit=50) - release_radar_tracks = [] + # Determine available sources + spotify_available = self.spotify_client and self.spotify_client.is_spotify_authenticated() + itunes_client = iTunesClient() - if recent_albums: - # Group albums by artist for variety - albums_by_artist = {} - for album in recent_albums: - artist = album['artist_name'] - if artist not in albums_by_artist: - albums_by_artist[artist] = [] - albums_by_artist[artist].append(album) + # Process each available source + sources_to_process = ['itunes'] # iTunes always available + if spotify_available: + sources_to_process.append('spotify') - # Get tracks from each album, grouped by artist - # IMPROVED: Add popularity scoring for smarter selection - artist_tracks = {} - artist_track_data = {} # Store full track data with scores + logger.info(f"Curating playlists for sources: {sources_to_process}") - for artist, albums in albums_by_artist.items(): - artist_tracks[artist] = [] - artist_track_data[artist] = [] + for source in sources_to_process: + logger.info(f"Curating Release Radar for {source}...") + + # 1. Curate Release Radar - 50 tracks from recent albums + recent_albums = self.database.get_discovery_recent_albums(limit=50, source=source) + release_radar_tracks = [] + + if recent_albums: + # Group albums by artist for variety + albums_by_artist = {} + for album in recent_albums: + artist = album['artist_name'] + if artist not in albums_by_artist: + albums_by_artist[artist] = [] + albums_by_artist[artist].append(album) + + # Get tracks from each album + artist_track_data = {} + + for artist, albums in albums_by_artist.items(): + artist_track_data[artist] = [] + + for album in albums: + try: + # Get album data from appropriate source + album_id = album.get('album_spotify_id') if source == 'spotify' else album.get('album_itunes_id') + if not album_id: + continue + + if source == 'spotify': + album_data = self.spotify_client.get_album(album_id) + else: + album_data = itunes_client.get_album(album_id) + + if not album_data or 'tracks' not in album_data: + continue - for album in albums: - try: - album_data = self.spotify_client.get_album(album['album_spotify_id']) - if album_data and 'tracks' in album_data: # Calculate days since release for recency score - days_old = 14 # Default + days_old = 14 try: release_date_str = album.get('release_date', '') if release_date_str and len(release_date_str) >= 10: @@ -2153,166 +2211,140 @@ class WatchlistScanner: except: pass - for track in album_data['tracks']['items']: - track_id = track['id'] + for track in album_data['tracks'].get('items', []): + track_id = track.get('id') + if not track_id: + continue - # Calculate track score (Spotify-style) - # Score factors: recency (50%), popularity (30%), singles bonus (20%) - recency_score = max(0, 100 - (days_old * 7)) # Newer = higher + # Calculate track score + recency_score = max(0, 100 - (days_old * 7)) popularity_score = track.get('popularity', album_data.get('popularity', 50)) is_single = album.get('album_type', 'album') == 'single' single_bonus = 20 if is_single else 0 - total_score = (recency_score * 0.5) + (popularity_score * 0.3) + single_bonus - artist_tracks[artist].append(track_id) - - # Store full track data with score for sorting - # Only include album metadata (not full album with all tracks) full_track = { 'id': track_id, - 'name': track['name'], - 'artists': track.get('artists', []), + 'name': track.get('name', 'Unknown'), + 'artists': track.get('artists', [{'name': artist}]), 'album': { - 'id': album_data['id'], + 'id': album_data.get('id', ''), 'name': album_data.get('name', 'Unknown Album'), 'images': album_data.get('images', []), 'release_date': album_data.get('release_date', ''), 'album_type': album_data.get('album_type', 'album'), - 'total_tracks': album_data.get('total_tracks', 0) }, 'duration_ms': track.get('duration_ms', 0), 'popularity': popularity_score, 'score': total_score, - 'days_old': days_old + 'source': source } artist_track_data[artist].append(full_track) + except Exception as e: + logger.debug(f"Error processing album for {artist}: {e}") + continue + + # Balance by artist - max 6 tracks per artist + balanced_track_data = [] + for artist, tracks in artist_track_data.items(): + sorted_tracks = sorted(tracks, key=lambda t: t['score'], reverse=True) + balanced_track_data.extend(sorted_tracks[:6]) + + # Sort by score and shuffle + balanced_track_data.sort(key=lambda t: t['score'], reverse=True) + top_tracks = balanced_track_data[:75] + random.shuffle(top_tracks) + + # Take final 50 tracks + release_radar_tracks = [track['id'] for track in top_tracks[:50]] + + # Add tracks to discovery pool + for track_data in top_tracks[:50]: + try: + artist_name = track_data['artists'][0].get('name', 'Unknown') if track_data['artists'] else 'Unknown' + formatted_track = { + 'track_name': track_data['name'], + 'artist_name': artist_name, + 'album_name': track_data['album'].get('name', 'Unknown'), + 'album_cover_url': track_data['album']['images'][0]['url'] if track_data['album'].get('images') else None, + 'duration_ms': track_data.get('duration_ms', 0), + 'popularity': track_data.get('popularity', 0), + 'release_date': track_data['album'].get('release_date', ''), + 'is_new_release': True, + 'track_data_json': track_data, + 'artist_genres': [] + } + if source == 'spotify': + formatted_track['spotify_track_id'] = track_data['id'] + formatted_track['spotify_album_id'] = track_data['album'].get('id', '') + else: + formatted_track['itunes_track_id'] = track_data['id'] + formatted_track['itunes_album_id'] = track_data['album'].get('id', '') + + self.database.add_to_discovery_pool(formatted_track, source=source) except Exception as e: continue - # IMPROVED: Balance by artist with popularity weighting - max 6 tracks per artist - balanced_tracks = [] - balanced_track_data = [] + # Save with source suffix for multi-source support + playlist_key = f'release_radar_{source}' + self.database.save_curated_playlist(playlist_key, release_radar_tracks) + logger.info(f"Release Radar ({source}) curated: {len(release_radar_tracks)} tracks") - for artist, track_data in artist_track_data.items(): - # Sort by score and take top 6 (not random) - sorted_tracks = sorted(track_data, key=lambda t: t['score'], reverse=True) - selected_tracks = sorted_tracks[:6] + # 2. Curate Discovery Weekly - 50 tracks from discovery pool + logger.info(f"Curating Discovery Weekly for {source}...") + discovery_tracks = self.database.get_discovery_pool_tracks(limit=2000, new_releases_only=False, source=source) - # Add selected tracks + discovery_weekly_tracks = [] + if discovery_tracks: + # Separate tracks by popularity tiers + popular_picks = [] + balanced_mix = [] + deep_cuts = [] + + for track in discovery_tracks: + popularity = track.popularity if hasattr(track, 'popularity') else 50 + if popularity >= 60: + popular_picks.append(track) + elif popularity >= 40: + balanced_mix.append(track) + else: + deep_cuts.append(track) + + logger.info(f"Discovery pool ({source}): {len(popular_picks)} popular, {len(balanced_mix)} mid-tier, {len(deep_cuts)} deep cuts") + + # Balanced selection + random.shuffle(popular_picks) + random.shuffle(balanced_mix) + random.shuffle(deep_cuts) + + selected_tracks = [] + selected_tracks.extend(popular_picks[:20]) + selected_tracks.extend(balanced_mix[:20]) + selected_tracks.extend(deep_cuts[:10]) + random.shuffle(selected_tracks) + + # Extract appropriate track IDs based on source for track in selected_tracks: - balanced_tracks.append(track['id']) - balanced_track_data.append(track) + if source == 'spotify' and track.spotify_track_id: + discovery_weekly_tracks.append(track.spotify_track_id) + elif source == 'itunes' and track.itunes_track_id: + discovery_weekly_tracks.append(track.itunes_track_id) - # IMPROVED: Sort by score first, then shuffle for variety - balanced_track_data.sort(key=lambda t: t['score'], reverse=True) + playlist_key = f'discovery_weekly_{source}' + self.database.save_curated_playlist(playlist_key, discovery_weekly_tracks) + logger.info(f"Discovery Weekly ({source}) curated: {len(discovery_weekly_tracks)} tracks") - # Take top 75, then shuffle for final randomization (prevents album grouping) - top_tracks = balanced_track_data[:75] - random.shuffle(top_tracks) + # Also save without suffix for backward compatibility (use active source) + active_source = 'spotify' if spotify_available else 'itunes' + release_radar_key = f'release_radar_{active_source}' + discovery_weekly_key = f'discovery_weekly_{active_source}' - # Take final 50 tracks - release_radar_tracks = [track['id'] for track in top_tracks[:50]] - release_radar_track_data = top_tracks[:50] - - # Add Release Radar tracks to discovery pool so they're available for fast lookup - logger.info(f"Adding {len(release_radar_track_data)} Release Radar tracks to discovery pool...") - - # Cache genres by artist_id to avoid duplicate API calls - artist_genres_cache = {} - - for track_data in release_radar_track_data: - try: - # Fetch artist genres (with caching) - artist_genres = [] - if track_data['artists'] and len(track_data['artists']) > 0: - artist_id = track_data['artists'][0]['id'] - - if artist_id in artist_genres_cache: - artist_genres = artist_genres_cache[artist_id] - else: - try: - artist_data = self.spotify_client.get_artist(artist_id) - if artist_data and 'genres' in artist_data: - artist_genres = artist_data['genres'] - artist_genres_cache[artist_id] = artist_genres - except Exception as e: - logger.debug(f"Could not fetch genres for artist {artist_id}: {e}") - - # Format track data for discovery pool (expects specific structure) - formatted_track = { - 'spotify_track_id': track_data['id'], - 'spotify_album_id': track_data['album'].get('id', ''), - 'spotify_artist_id': track_data['artists'][0]['id'] if track_data['artists'] else '', - 'track_name': track_data['name'], - 'artist_name': track_data['artists'][0]['name'] if track_data['artists'] else 'Unknown', - 'album_name': track_data['album'].get('name', 'Unknown'), - 'album_cover_url': track_data['album']['images'][0]['url'] if track_data['album'].get('images') else None, - 'duration_ms': track_data.get('duration_ms', 0), - 'popularity': track_data.get('popularity', 0), - 'release_date': track_data['album'].get('release_date', ''), - 'is_new_release': True, - 'track_data_json': track_data, - 'artist_genres': artist_genres - } - self.database.add_to_discovery_pool(formatted_track) - except Exception as e: - logger.warning(f"Failed to add track {track_data['name']} to discovery pool: {e}") - continue - - self.database.save_curated_playlist('release_radar', release_radar_tracks) - logger.info(f"Release Radar curated: {len(release_radar_tracks)} tracks") - - # 2. Curate Discovery Weekly - 50 tracks from full discovery pool - # IMPROVED: Spotify-style algorithm with balanced mix of popular, mid-tier, and deep cuts - logger.info("Curating Discovery Weekly playlist...") - discovery_tracks = self.database.get_discovery_pool_tracks(limit=2000, new_releases_only=False) - - discovery_weekly_tracks = [] - if discovery_tracks: - # Separate tracks by popularity tiers for balanced selection - popular_picks = [] # popularity >= 60 - balanced_mix = [] # 40 <= popularity < 60 - deep_cuts = [] # popularity < 40 - - for track in discovery_tracks: - popularity = track.popularity if hasattr(track, 'popularity') else 50 - - if popularity >= 60: - popular_picks.append(track) - elif popularity >= 40: - balanced_mix.append(track) - else: - deep_cuts.append(track) - - logger.info(f"Discovery pool breakdown: {len(popular_picks)} popular, {len(balanced_mix)} mid-tier, {len(deep_cuts)} deep cuts") - - # Create balanced playlist (Spotify-style distribution) - # 40% popular picks (20 tracks) - # 40% balanced mid-tier (20 tracks) - # 20% deep cuts (10 tracks) - selected_tracks = [] - - # Randomly select from each tier - random.shuffle(popular_picks) - random.shuffle(balanced_mix) - random.shuffle(deep_cuts) - - selected_tracks.extend(popular_picks[:20]) # 20 popular - selected_tracks.extend(balanced_mix[:20]) # 20 mid-tier - selected_tracks.extend(deep_cuts[:10]) # 10 deep cuts - - # Shuffle final selection for variety - random.shuffle(selected_tracks) - - # Extract track IDs - discovery_weekly_tracks = [track.spotify_track_id for track in selected_tracks] - - logger.info(f"Discovery Weekly composition: {len(popular_picks[:20])} popular + {len(balanced_mix[:20])} mid-tier + {len(deep_cuts[:10])} deep cuts = {len(discovery_weekly_tracks)} total") - - self.database.save_curated_playlist('discovery_weekly', discovery_weekly_tracks) - logger.info(f"Discovery Weekly curated: {len(discovery_weekly_tracks)} tracks") + # Copy active source playlists to non-suffixed keys + release_radar_ids = self.database.get_curated_playlist(release_radar_key) or [] + discovery_weekly_ids = self.database.get_curated_playlist(discovery_weekly_key) or [] + self.database.save_curated_playlist('release_radar', release_radar_ids) + self.database.save_curated_playlist('discovery_weekly', discovery_weekly_ids) logger.info("Playlist curation complete") diff --git a/web_server.py b/web_server.py index f2826a02..fe96c049 100644 --- a/web_server.py +++ b/web_server.py @@ -18337,8 +18337,10 @@ def get_discover_release_radar(): # Determine active source - release radar works with any source now active_source = _get_active_discovery_source() - # Try to get curated playlist first - curated_track_ids = database.get_curated_playlist('release_radar') + # Try source-specific playlist first, then fall back to generic + curated_track_ids = database.get_curated_playlist(f'release_radar_{active_source}') + if not curated_track_ids: + curated_track_ids = database.get_curated_playlist('release_radar') if curated_track_ids: # Use curated selection - fetch track data from discovery pool filtered by source @@ -18398,8 +18400,10 @@ def get_discover_weekly(): # Determine active source active_source = _get_active_discovery_source() - # Try to get curated playlist first - curated_track_ids = database.get_curated_playlist('discovery_weekly') + # Try source-specific playlist first, then fall back to generic + curated_track_ids = database.get_curated_playlist(f'discovery_weekly_{active_source}') + if not curated_track_ids: + curated_track_ids = database.get_curated_playlist('discovery_weekly') if curated_track_ids: # Use curated selection - fetch track data from discovery pool filtered by source From 84e6b01cc67e68d03004fffd28e87be6ab888f15 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sat, 24 Jan 2026 14:17:09 -0800 Subject: [PATCH 09/20] Add iTunes fallback to Tidal and Beatport discovery workers - Update _search_spotify_for_tidal_track to accept use_spotify and itunes_client parameters for dual-source support - Update _run_tidal_discovery_worker to check is_spotify_authenticated() and fall back to iTunes when Spotify unavailable - Update _run_beatport_discovery_worker with same iTunes fallback pattern - Add discovery_source field to state and results for frontend awareness - Activity messages now indicate which provider (SPOTIFY/ITUNES) completed discovery All four discovery modals (YouTube, ListenBrainz, Tidal, Beatport) now support the dual-source architecture: Spotify preferred, iTunes fallback. --- web_server.py | 967 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 627 insertions(+), 340 deletions(-) diff --git a/web_server.py b/web_server.py index fe96c049..c18871be 100644 --- a/web_server.py +++ b/web_server.py @@ -14721,30 +14721,47 @@ def update_tidal_playlist_phase(playlist_id): def _run_tidal_discovery_worker(playlist_id): - """Background worker for Tidal Spotify discovery process (like sync.py)""" + """Background worker for Tidal discovery process (Spotify preferred, iTunes fallback)""" try: state = tidal_discovery_states[playlist_id] playlist = state['playlist'] - - print(f"🎡 Starting Tidal Spotify discovery for: {playlist.name}") - + + # Determine which provider to use + use_spotify = spotify_client and spotify_client.is_spotify_authenticated() + discovery_source = 'spotify' if use_spotify else 'itunes' + + # Initialize iTunes client if needed + itunes_client_instance = None + if not use_spotify: + from core.itunes_client import iTunesClient + itunes_client_instance = iTunesClient() + + print(f"🎡 Starting Tidal discovery for: {playlist.name} (using {discovery_source.upper()})") + + # Store discovery source in state for frontend + state['discovery_source'] = discovery_source + # Import matching engine for validation (like sync.py) from core.matching_engine import MusicMatchingEngine matching_engine = MusicMatchingEngine() - + successful_discoveries = 0 - + for i, tidal_track in enumerate(playlist.tracks): if state.get('cancelled', False): break - + try: - print(f"πŸ” [{i+1}/{len(playlist.tracks)}] Searching: {tidal_track.name} by {', '.join(tidal_track.artists)}") - - # Use the same search logic as sync.py TidalSpotifyDiscoveryWorker - spotify_track = _search_spotify_for_tidal_track(tidal_track) - - # Create result entry + print(f"πŸ” [{i+1}/{len(playlist.tracks)}] Searching {discovery_source.upper()}: {tidal_track.name} by {', '.join(tidal_track.artists)}") + + # Use the search function with appropriate provider + track_result = _search_spotify_for_tidal_track( + tidal_track, + use_spotify=use_spotify, + itunes_client=itunes_client_instance + ) + + # Create result entry - use 'match_data' as generic key for both providers result = { 'tidal_track': { 'id': tidal_track.id, @@ -14753,47 +14770,66 @@ def _run_tidal_discovery_worker(playlist_id): 'album': getattr(tidal_track, 'album', 'Unknown Album'), 'duration_ms': getattr(tidal_track, 'duration_ms', 0), }, - 'spotify_data': None, - 'status': 'not_found' + 'spotify_data': None, # Keep for backwards compatibility + 'match_data': None, # Generic field for any provider + 'status': 'not_found', + 'discovery_source': discovery_source } - - if isinstance(spotify_track, tuple): - # Function now returns (Track, raw_data) - track_obj, raw_track_data = spotify_track + + if use_spotify and isinstance(track_result, tuple): + # Spotify: Function returns (Track, raw_data) + track_obj, raw_track_data = track_result # Use full album object from raw API response album_obj = raw_track_data.get('album', {}) if raw_track_data else {} - result['spotify_data'] = { + match_data = { 'id': track_obj.id, 'name': track_obj.name, 'artists': track_obj.artists, # Already a list of strings 'album': album_obj, # Full album object with images 'duration_ms': track_obj.duration_ms, - 'external_urls': track_obj.external_urls + 'external_urls': track_obj.external_urls, + 'source': 'spotify' } + result['spotify_data'] = match_data # Backwards compatibility + result['match_data'] = match_data result['status'] = 'found' successful_discoveries += 1 state['spotify_matches'] = successful_discoveries - elif spotify_track: - # Fallback for old format (shouldn't happen after update) - result['spotify_data'] = { - 'id': spotify_track.id, - 'name': spotify_track.name, - 'artists': spotify_track.artists, - 'album': {'name': spotify_track.album, 'album_type': 'album', 'images': []}, - 'duration_ms': spotify_track.duration_ms, - 'external_urls': spotify_track.external_urls - } + + elif not use_spotify and track_result and isinstance(track_result, dict): + # iTunes: Function returns a dict with track data + match_data = track_result + match_data['source'] = 'itunes' + result['spotify_data'] = match_data # Use same field for frontend compatibility + result['match_data'] = match_data result['status'] = 'found' successful_discoveries += 1 state['spotify_matches'] = successful_discoveries - + + elif use_spotify and track_result: + # Spotify fallback for old format (shouldn't happen after update) + match_data = { + 'id': track_result.id, + 'name': track_result.name, + 'artists': track_result.artists, + 'album': {'name': track_result.album, 'album_type': 'album', 'images': []}, + 'duration_ms': track_result.duration_ms, + 'external_urls': track_result.external_urls, + 'source': 'spotify' + } + result['spotify_data'] = match_data + result['match_data'] = match_data + result['status'] = 'found' + successful_discoveries += 1 + state['spotify_matches'] = successful_discoveries + state['discovery_results'].append(result) state['discovery_progress'] = int(((i + 1) / len(playlist.tracks)) * 100) - - # Add delay between requests (like sync.py) + + # Add delay between requests time.sleep(0.1) - + except Exception as e: print(f"❌ Error processing track {i+1}: {e}") # Add error result @@ -14803,32 +14839,49 @@ def _run_tidal_discovery_worker(playlist_id): 'artists': tidal_track.artists or [], }, 'spotify_data': None, + 'match_data': None, 'status': 'error', - 'error': str(e) + 'error': str(e), + 'discovery_source': discovery_source } state['discovery_results'].append(result) state['discovery_progress'] = int(((i + 1) / len(playlist.tracks)) * 100) - + # Mark as complete state['phase'] = 'discovered' state['status'] = 'discovered' state['discovery_progress'] = 100 - + # Add activity for discovery completion - add_activity_item("βœ…", "Tidal Discovery Complete", f"'{playlist.name}' - {successful_discoveries}/{len(playlist.tracks)} tracks found", "Now") - - print(f"βœ… Tidal discovery complete: {successful_discoveries}/{len(playlist.tracks)} tracks found") - + source_label = discovery_source.upper() + add_activity_item("βœ…", f"Tidal Discovery Complete ({source_label})", f"'{playlist.name}' - {successful_discoveries}/{len(playlist.tracks)} tracks found", "Now") + + print(f"βœ… Tidal discovery complete ({source_label}): {successful_discoveries}/{len(playlist.tracks)} tracks found") + except Exception as e: print(f"❌ Error in Tidal discovery worker: {e}") state['phase'] = 'error' state['status'] = f'error: {str(e)}' -def _search_spotify_for_tidal_track(tidal_track): - """Search Spotify for a Tidal track using matching_engine for better accuracy""" - if not spotify_client or not spotify_client.is_authenticated(): - return None +def _search_spotify_for_tidal_track(tidal_track, use_spotify=True, itunes_client=None): + """Search Spotify/iTunes for a Tidal track using matching_engine for better accuracy + + Args: + tidal_track: The Tidal track to search for + use_spotify: If True, use Spotify; if False, use iTunes + itunes_client: iTunes client instance (required when use_spotify=False) + + Returns: + For Spotify: (Track, raw_data) tuple or None + For iTunes: dict with track data or None + """ + if use_spotify: + if not spotify_client or not spotify_client.is_authenticated(): + return None + else: + if not itunes_client: + return None try: # Get track info @@ -14839,8 +14892,9 @@ def _search_spotify_for_tidal_track(tidal_track): return None artist_name = artists[0] # Use primary artist + source_name = "Spotify" if use_spotify else "iTunes" - print(f"πŸ” Tidal track: '{artist_name}' - '{track_name}'") + print(f"πŸ” Tidal track: '{artist_name}' - '{track_name}' (searching {source_name})") # Use matching engine to generate search queries (with fallback) try: @@ -14869,62 +14923,113 @@ def _search_spotify_for_tidal_track(tidal_track): for query_idx, search_query in enumerate(search_queries): try: - print(f"πŸ” Tidal query {query_idx + 1}/{len(search_queries)}: {search_query}") + print(f"πŸ” Tidal query {query_idx + 1}/{len(search_queries)}: {search_query} ({source_name})") - # Get raw Spotify API response to access full album object with images - raw_results = spotify_client.sp.search(q=search_query, type='track', limit=5) - if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']: - continue + if use_spotify: + # SPOTIFY PATH: Get raw Spotify API response to access full album object with images + raw_results = spotify_client.sp.search(q=search_query, type='track', limit=5) + if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']: + continue - # Also get Track objects for matching logic - results = spotify_client.search_tracks(search_query, limit=5) + # Also get Track objects for matching logic + results = spotify_client.search_tracks(search_query, limit=5) - if not results: - continue + if not results: + continue - # Score each result using matching engine - for idx, result in enumerate(results): - raw_track = raw_results['tracks']['items'][idx] if idx < len(raw_results['tracks']['items']) else None - try: - # Calculate confidence using matching engine's similarity scoring (with fallback) + # Score each result using matching engine + for idx, result in enumerate(results): + raw_track = raw_results['tracks']['items'][idx] if idx < len(raw_results['tracks']['items']) else None try: - artist_confidence = 0.0 - if result.artists: - # Get best artist match confidence - for result_artist in result.artists: + # Calculate confidence using matching engine's similarity scoring (with fallback) + try: + artist_confidence = 0.0 + if result.artists: + # Get best artist match confidence + for result_artist in result.artists: + artist_sim = matching_engine.similarity_score( + matching_engine.normalize_string(artist_name), + matching_engine.normalize_string(result_artist) + ) + artist_confidence = max(artist_confidence, artist_sim) + + # Calculate title confidence + title_confidence = matching_engine.similarity_score( + matching_engine.normalize_string(track_name), + matching_engine.normalize_string(result.name) + ) + + # Combined confidence (equal weighting for Tidal clean data) + combined_confidence = (artist_confidence * 0.5 + title_confidence * 0.5) + except Exception as e: + print(f"⚠️ Matching engine scoring failed for Tidal, using first match: {e}") + # Fallback: just take the first result if matching engine fails + combined_confidence = 1.0 # Set high to accept this match + best_match = result + break + + print(f"πŸ” Tidal candidate: '{result.artists[0]}' - '{result.name}' (confidence: {combined_confidence:.3f})") + + # Update best match if this is better + if combined_confidence > best_confidence and combined_confidence >= min_confidence: + best_confidence = combined_confidence + best_match = result + best_match_raw = raw_track # Store raw data with full album object + print(f"βœ… New best Tidal match: {result.artists[0]} - {result.name} (confidence: {combined_confidence:.3f})") + + except Exception as e: + print(f"❌ Error processing Tidal search result: {e}") + continue + + else: + # ITUNES PATH: Search using iTunes client + # For iTunes, use a simpler query format + simple_query = f"{artist_name} {track_name}" + itunes_results = itunes_client.search_tracks(simple_query, limit=5) + + if not itunes_results: + continue + + # Score each iTunes result + for result in itunes_results: + try: + # Calculate confidence using matching engine + try: + artist_confidence = 0.0 + result_artist = result.artist if hasattr(result, 'artist') else result.get('artist', '') + if result_artist: artist_sim = matching_engine.similarity_score( matching_engine.normalize_string(artist_name), matching_engine.normalize_string(result_artist) ) - artist_confidence = max(artist_confidence, artist_sim) + artist_confidence = artist_sim - # Calculate title confidence - title_confidence = matching_engine.similarity_score( - matching_engine.normalize_string(track_name), - matching_engine.normalize_string(result.name) - ) + # Calculate title confidence + result_name = result.name if hasattr(result, 'name') else result.get('name', '') + title_confidence = matching_engine.similarity_score( + matching_engine.normalize_string(track_name), + matching_engine.normalize_string(result_name) + ) + + combined_confidence = (artist_confidence * 0.5 + title_confidence * 0.5) + except Exception as e: + print(f"⚠️ Matching engine scoring failed for iTunes Tidal, using first match: {e}") + combined_confidence = 1.0 + best_match = result + break + + result_artist_display = result.artist if hasattr(result, 'artist') else result.get('artist', 'Unknown') + result_name_display = result.name if hasattr(result, 'name') else result.get('name', 'Unknown') + print(f"πŸ” iTunes Tidal candidate: '{result_artist_display}' - '{result_name_display}' (confidence: {combined_confidence:.3f})") + + if combined_confidence > best_confidence and combined_confidence >= min_confidence: + best_confidence = combined_confidence + best_match = result + print(f"βœ… New best iTunes Tidal match: {result_artist_display} - {result_name_display} (confidence: {combined_confidence:.3f})") - # Combined confidence (equal weighting for Tidal clean data) - combined_confidence = (artist_confidence * 0.5 + title_confidence * 0.5) except Exception as e: - print(f"⚠️ Matching engine scoring failed for Tidal, using first match: {e}") - # Fallback: just take the first result if matching engine fails - combined_confidence = 1.0 # Set high to accept this match - best_match = result - break - - print(f"πŸ” Tidal candidate: '{result.artists[0]}' - '{result.name}' (confidence: {combined_confidence:.3f})") - - # Update best match if this is better - if combined_confidence > best_confidence and combined_confidence >= min_confidence: - best_confidence = combined_confidence - best_match = result - best_match_raw = raw_track # Store raw data with full album object - print(f"βœ… New best Tidal match: {result.artists[0]} - {result.name} (confidence: {combined_confidence:.3f})") - - except Exception as e: - print(f"❌ Error processing Tidal search result: {e}") - continue + print(f"❌ Error processing iTunes Tidal search result: {e}") + continue # If we found a very high confidence match, stop searching if best_confidence >= 0.9: @@ -14932,12 +15037,37 @@ def _search_spotify_for_tidal_track(tidal_track): break except Exception as e: - print(f"❌ Error in Tidal Spotify search for query '{search_query}': {e}") + print(f"❌ Error in Tidal {source_name} search for query '{search_query}': {e}") continue if best_match: - print(f"βœ… Final Tidal match: {best_match.artists[0]} - {best_match.name} (confidence: {best_confidence:.3f})") - return (best_match, best_match_raw) # Return both Track object and raw data + if use_spotify: + print(f"βœ… Final Tidal Spotify match: {best_match.artists[0]} - {best_match.name} (confidence: {best_confidence:.3f})") + return (best_match, best_match_raw) # Return both Track object and raw data + else: + # For iTunes, return a dict with normalized data + result_artist = best_match.artist if hasattr(best_match, 'artist') else best_match.get('artist', 'Unknown') + result_name = best_match.name if hasattr(best_match, 'name') else best_match.get('name', 'Unknown') + print(f"βœ… Final Tidal iTunes match: {result_artist} - {result_name} (confidence: {best_confidence:.3f})") + + # Build iTunes result dict with album info + album_name = best_match.album if hasattr(best_match, 'album') else best_match.get('album', 'Unknown Album') + artwork_url = best_match.artwork_url if hasattr(best_match, 'artwork_url') else best_match.get('artwork_url', '') + track_id = best_match.id if hasattr(best_match, 'id') else best_match.get('id', '') + duration_ms = best_match.duration_ms if hasattr(best_match, 'duration_ms') else best_match.get('duration_ms', 0) + + return { + 'id': track_id, + 'name': result_name, + 'artists': [result_artist], + 'album': { + 'name': album_name, + 'album_type': 'album', + 'images': [{'url': artwork_url, 'height': 300, 'width': 300}] if artwork_url else [] + }, + 'duration_ms': duration_ms, + 'source': 'itunes' + } else: print(f"❌ No suitable Tidal match found (best confidence was {best_confidence:.3f}, required {min_confidence:.3f})") return None @@ -15312,34 +15442,39 @@ def update_youtube_discovery_match(): def _run_youtube_discovery_worker(url_hash): - """Background worker for YouTube Spotify discovery process""" + """Background worker for YouTube music discovery process (Spotify preferred, iTunes fallback)""" try: state = youtube_playlist_states[url_hash] playlist = state['playlist'] tracks = playlist['tracks'] + + # Determine which provider to use (Spotify preferred, iTunes fallback) + use_spotify = spotify_client and spotify_client.is_spotify_authenticated() + discovery_source = 'spotify' if use_spotify else 'itunes' + + # Get iTunes client for fallback + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() + + print(f"πŸ” Starting {discovery_source} discovery for {len(tracks)} YouTube tracks...") + + # Store the discovery source in state + state['discovery_source'] = discovery_source - print(f"πŸ” Starting Spotify discovery for {len(tracks)} YouTube tracks...") - - if not spotify_client or not spotify_client.is_authenticated(): - print("❌ Spotify client not authenticated") - state['status'] = 'error' - state['phase'] = 'fresh' - return - - # Process each track for Spotify discovery + # Process each track for discovery for i, track in enumerate(tracks): try: # Update progress state['discovery_progress'] = int((i / len(tracks)) * 100) - - # Search for track on Spotify using cleaned data + + # Search for track using active provider cleaned_title = track['name'] cleaned_artist = track['artists'][0] if track['artists'] else 'Unknown Artist' - - print(f"πŸ” Searching Spotify for: '{cleaned_artist}' - '{cleaned_title}'") + + print(f"πŸ” Searching {discovery_source} for: '{cleaned_artist}' - '{cleaned_title}'") # Try multiple search strategies using matching_engine for better accuracy - spotify_track = None + matched_track = None best_confidence = 0.0 min_confidence = 0.6 # Keep same threshold as before @@ -15358,33 +15493,42 @@ def _run_youtube_discovery_worker(url_hash): # Fallback to original simple query search_queries = [f"artist:{cleaned_artist} track:{cleaned_title}"] - # Store raw Spotify data for best match + # Store raw data for best match best_raw_track = None for query_idx, search_query in enumerate(search_queries): try: print(f"πŸ” YouTube query {query_idx + 1}/{len(search_queries)}: {search_query}") - # Get raw Spotify API response to access full album object with images - raw_results = spotify_client.sp.search(q=search_query, type='track', limit=5) - if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']: - continue + # Search using appropriate provider + raw_results = None + search_results = None - spotify_results = spotify_client.search_tracks(search_query, limit=5) + if use_spotify: + # Get raw Spotify API response to access full album object with images + raw_results = spotify_client.sp.search(q=search_query, type='track', limit=5) + if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']: + continue + search_results = spotify_client.search_tracks(search_query, limit=5) + else: + # Use iTunes search + search_results = itunes_client.search_tracks(search_query, limit=5) - if not spotify_results: + if not search_results: continue # Score each result using matching engine - for result_idx, spotify_result in enumerate(spotify_results): - raw_track = raw_results['tracks']['items'][result_idx] if result_idx < len(raw_results['tracks']['items']) else None + for result_idx, search_result in enumerate(search_results): + raw_track = None + if use_spotify and raw_results: + raw_track = raw_results['tracks']['items'][result_idx] if result_idx < len(raw_results['tracks']['items']) else None try: # Calculate confidence using matching engine's similarity scoring (with fallback) try: artist_confidence = 0.0 - if spotify_result.artists: + if search_result.artists: # Get best artist match confidence - for result_artist in spotify_result.artists: + for result_artist in search_result.artists: artist_sim = matching_engine.similarity_score( matching_engine.normalize_string(cleaned_artist), matching_engine.normalize_string(result_artist) @@ -15394,7 +15538,7 @@ def _run_youtube_discovery_worker(url_hash): # Calculate title confidence title_confidence = matching_engine.similarity_score( matching_engine.normalize_string(cleaned_title), - matching_engine.normalize_string(spotify_result.name) + matching_engine.normalize_string(search_result.name) ) # Combined confidence (70% title, 30% artist - same as original) @@ -15417,18 +15561,18 @@ def _run_youtube_discovery_worker(url_hash): union = len(set1.union(set2)) return intersection / union if union > 0 else 0 - title_score = _calculate_similarity_fallback(cleaned_title, spotify_result.name) - artist_score = _calculate_similarity_fallback(cleaned_artist, spotify_result.artists[0] if spotify_result.artists else "") + title_score = _calculate_similarity_fallback(cleaned_title, search_result.name) + artist_score = _calculate_similarity_fallback(cleaned_artist, search_result.artists[0] if search_result.artists else "") combined_confidence = (title_score * 0.7) + (artist_score * 0.3) - print(f"πŸ” YouTube candidate: '{spotify_result.artists[0]}' - '{spotify_result.name}' (confidence: {combined_confidence:.3f})") + print(f"πŸ” YouTube candidate: '{search_result.artists[0]}' - '{search_result.name}' (confidence: {combined_confidence:.3f})") # Update best match if this is better if combined_confidence > best_confidence and combined_confidence >= min_confidence: best_confidence = combined_confidence - spotify_track = spotify_result - best_raw_track = raw_track # Store raw data with full album object - print(f"βœ… New best YouTube match: {spotify_result.artists[0]} - {spotify_result.name} (confidence: {combined_confidence:.3f})") + matched_track = search_result + best_raw_track = raw_track # Store raw data with full album object (Spotify only) + print(f"βœ… New best YouTube match: {search_result.artists[0]} - {search_result.name} (confidence: {combined_confidence:.3f})") except Exception as e: print(f"❌ Error processing YouTube search result: {e}") @@ -15443,61 +15587,78 @@ def _run_youtube_discovery_worker(url_hash): print(f"❌ Error in YouTube search for query '{search_query}': {e}") continue - if spotify_track: - print(f"βœ… Strategy 1 YouTube match: {spotify_track.artists[0]} - {spotify_track.name} (confidence: {best_confidence:.3f})") - + if matched_track: + print(f"βœ… Strategy 1 YouTube match: {matched_track.artists[0]} - {matched_track.name} (confidence: {best_confidence:.3f})") + # Strategy 2: Swapped search (if first failed) - keep simple for fallback - if not spotify_track: + if not matched_track: print("πŸ”„ YouTube Strategy 2: Trying swapped search (artist/title reversed)") query = f"artist:{cleaned_title} track:{cleaned_artist}" - spotify_results = spotify_client.search_tracks(query, limit=3) - if spotify_results: - spotify_track = spotify_results[0] - print(f"βœ… Strategy 2 YouTube match (swapped): {spotify_track.artists[0]} - {spotify_track.name}") + if use_spotify: + fallback_results = spotify_client.search_tracks(query, limit=3) + else: + fallback_results = itunes_client.search_tracks(query, limit=3) + if fallback_results: + matched_track = fallback_results[0] + print(f"βœ… Strategy 2 YouTube match (swapped): {matched_track.artists[0]} - {matched_track.name}") # Strategy 3: Raw data search (if still failed) - keep simple for fallback - if not spotify_track: + if not matched_track: raw_title = track['raw_title'] raw_artist = track['raw_artist'] print(f"πŸ”„ YouTube Strategy 3: Trying raw data search: '{raw_artist} {raw_title}'") query = f"{raw_artist} {raw_title}" - spotify_results = spotify_client.search_tracks(query, limit=3) - if spotify_results: - spotify_track = spotify_results[0] - print(f"βœ… Strategy 3 YouTube match (raw): {spotify_track.artists[0]} - {spotify_track.name}") - + if use_spotify: + fallback_results = spotify_client.search_tracks(query, limit=3) + else: + fallback_results = itunes_client.search_tracks(query, limit=3) + if fallback_results: + matched_track = fallback_results[0] + print(f"βœ… Strategy 3 YouTube match (raw): {matched_track.artists[0]} - {matched_track.name}") + # Create result entry result = { 'index': i, 'yt_track': cleaned_title, 'yt_artist': cleaned_artist, - 'status': 'βœ… Found' if spotify_track else '❌ Not Found', - 'status_class': 'found' if spotify_track else 'not-found', - 'spotify_track': spotify_track.name if spotify_track else '', - 'spotify_artist': spotify_track.artists[0] if spotify_track else '', - 'spotify_album': spotify_track.album if spotify_track else '', - 'duration': f"{track['duration_ms'] // 60000}:{(track['duration_ms'] % 60000) // 1000:02d}" if track['duration_ms'] else '0:00' + 'status': 'βœ… Found' if matched_track else '❌ Not Found', + 'status_class': 'found' if matched_track else 'not-found', + 'spotify_track': matched_track.name if matched_track else '', + 'spotify_artist': matched_track.artists[0] if matched_track else '', + 'spotify_album': matched_track.album if matched_track else '', + 'duration': f"{track['duration_ms'] // 60000}:{(track['duration_ms'] % 60000) // 1000:02d}" if track['duration_ms'] else '0:00', + 'discovery_source': discovery_source } - - if spotify_track: - state['spotify_matches'] += 1 - # Use full album object from raw Spotify data if available - album_data = best_raw_track.get('album', {}) if best_raw_track else {} - if not album_data: - # Fallback to string album name - album_data = {'name': spotify_track.album, 'album_type': 'album', 'images': []} - result['spotify_data'] = { - 'id': spotify_track.id, - 'name': spotify_track.name, - 'artists': spotify_track.artists, - 'album': album_data, # Full album object with images - 'duration_ms': spotify_track.duration_ms + if matched_track: + state['spotify_matches'] += 1 # Keep key name for compatibility + + # Build album data based on provider + if use_spotify and best_raw_track: + album_data = best_raw_track.get('album', {}) + else: + # For iTunes or when raw data unavailable + album_data = { + 'name': matched_track.album, + 'album_type': 'album', + 'images': [{'url': matched_track.image_url}] if hasattr(matched_track, 'image_url') and matched_track.image_url else [] + } + + # Store track data with source info + result['matched_data'] = { + 'id': matched_track.id, + 'name': matched_track.name, + 'artists': matched_track.artists, + 'album': album_data, + 'duration_ms': matched_track.duration_ms, + 'source': discovery_source } + # Keep spotify_data for backward compatibility + result['spotify_data'] = result['matched_data'] state['discovery_results'].append(result) - - print(f" {'βœ…' if spotify_track else '❌'} Track {i+1}/{len(tracks)}: {result['status']}") + + print(f" {'βœ…' if matched_track else '❌'} Track {i+1}/{len(tracks)}: {result['status']}") except Exception as e: print(f"❌ Error processing track {i}: {e}") @@ -15522,9 +15683,10 @@ def _run_youtube_discovery_worker(url_hash): # Add activity for discovery completion playlist_name = playlist['name'] - add_activity_item("βœ…", "YouTube Discovery Complete", f"'{playlist_name}' - {state['spotify_matches']}/{len(tracks)} tracks found", "Now") - - print(f"βœ… YouTube discovery complete: {state['spotify_matches']}/{len(tracks)} tracks matched") + source_label = 'Spotify' if use_spotify else 'iTunes' + add_activity_item("βœ…", f"YouTube Discovery Complete ({source_label})", f"'{playlist_name}' - {state['spotify_matches']}/{len(tracks)} tracks found", "Now") + + print(f"βœ… YouTube discovery complete ({discovery_source}): {state['spotify_matches']}/{len(tracks)} tracks matched") except Exception as e: print(f"❌ Error in YouTube discovery worker: {e}") @@ -15532,21 +15694,26 @@ def _run_youtube_discovery_worker(url_hash): state['phase'] = 'fresh' def _run_listenbrainz_discovery_worker(playlist_mbid): - """Background worker for ListenBrainz Spotify discovery process""" + """Background worker for ListenBrainz music discovery process (Spotify preferred, iTunes fallback)""" try: state = listenbrainz_playlist_states[playlist_mbid] playlist = state['playlist'] tracks = playlist['tracks'] - print(f"πŸ” Starting Spotify discovery for {len(tracks)} ListenBrainz tracks...") + # Determine which provider to use (Spotify preferred, iTunes fallback) + use_spotify = spotify_client and spotify_client.is_spotify_authenticated() + discovery_source = 'spotify' if use_spotify else 'itunes' - if not spotify_client or not spotify_client.is_authenticated(): - print("❌ Spotify client not authenticated") - state['status'] = 'error' - state['phase'] = 'fresh' - return + # Get iTunes client for fallback + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() - # Process each track for Spotify discovery + print(f"πŸ” Starting {discovery_source} discovery for {len(tracks)} ListenBrainz tracks...") + + # Store the discovery source in state + state['discovery_source'] = discovery_source + + # Process each track for discovery for i, track in enumerate(tracks): try: # Update progress @@ -15558,10 +15725,10 @@ def _run_listenbrainz_discovery_worker(playlist_mbid): album_name = track.get('album_name', '') duration_ms = track.get('duration_ms', 0) - print(f"πŸ” Searching Spotify for: '{cleaned_artist}' - '{cleaned_title}'") + print(f"πŸ” Searching {discovery_source} for: '{cleaned_artist}' - '{cleaned_title}'") # Try multiple search strategies using matching_engine for better accuracy - spotify_track = None + matched_track = None best_confidence = 0.0 min_confidence = 0.6 # Keep same threshold as YouTube @@ -15580,33 +15747,42 @@ def _run_listenbrainz_discovery_worker(playlist_mbid): # Fallback to original simple query search_queries = [f"artist:{cleaned_artist} track:{cleaned_title}"] - # Store raw Spotify data for best match + # Store raw data for best match best_raw_track = None for query_idx, search_query in enumerate(search_queries): try: print(f"πŸ” ListenBrainz query {query_idx + 1}/{len(search_queries)}: {search_query}") - # Get raw Spotify API response to access full album object with images - raw_results = spotify_client.sp.search(q=search_query, type='track', limit=5) - if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']: - continue + # Search using appropriate provider + raw_results = None + search_results = None - spotify_results = spotify_client.search_tracks(search_query, limit=5) + if use_spotify: + # Get raw Spotify API response to access full album object with images + raw_results = spotify_client.sp.search(q=search_query, type='track', limit=5) + if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']: + continue + search_results = spotify_client.search_tracks(search_query, limit=5) + else: + # Use iTunes search + search_results = itunes_client.search_tracks(search_query, limit=5) - if not spotify_results: + if not search_results: continue # Score each result using matching engine - for result_idx, spotify_result in enumerate(spotify_results): - raw_track = raw_results['tracks']['items'][result_idx] if result_idx < len(raw_results['tracks']['items']) else None + for result_idx, search_result in enumerate(search_results): + raw_track = None + if use_spotify and raw_results: + raw_track = raw_results['tracks']['items'][result_idx] if result_idx < len(raw_results['tracks']['items']) else None try: # Calculate confidence using matching engine's similarity scoring (with fallback) try: artist_confidence = 0.0 - if spotify_result.artists: + if search_result.artists: # Get best artist match confidence - for result_artist in spotify_result.artists: + for result_artist in search_result.artists: artist_sim = matching_engine.similarity_score( matching_engine.normalize_string(cleaned_artist), matching_engine.normalize_string(result_artist) @@ -15616,7 +15792,7 @@ def _run_listenbrainz_discovery_worker(playlist_mbid): # Calculate title confidence title_confidence = matching_engine.similarity_score( matching_engine.normalize_string(cleaned_title), - matching_engine.normalize_string(spotify_result.name) + matching_engine.normalize_string(search_result.name) ) # Combined confidence (70% title, 30% artist - same as YouTube) @@ -15639,18 +15815,18 @@ def _run_listenbrainz_discovery_worker(playlist_mbid): union = len(set1.union(set2)) return intersection / union if union > 0 else 0 - title_score = _calculate_similarity_fallback(cleaned_title, spotify_result.name) - artist_score = _calculate_similarity_fallback(cleaned_artist, spotify_result.artists[0] if spotify_result.artists else "") + title_score = _calculate_similarity_fallback(cleaned_title, search_result.name) + artist_score = _calculate_similarity_fallback(cleaned_artist, search_result.artists[0] if search_result.artists else "") combined_confidence = (title_score * 0.7) + (artist_score * 0.3) - print(f"πŸ” ListenBrainz candidate: '{spotify_result.artists[0]}' - '{spotify_result.name}' (confidence: {combined_confidence:.3f})") + print(f"πŸ” ListenBrainz candidate: '{search_result.artists[0]}' - '{search_result.name}' (confidence: {combined_confidence:.3f})") # Update best match if this is better if combined_confidence > best_confidence and combined_confidence >= min_confidence: best_confidence = combined_confidence - spotify_track = spotify_result - best_raw_track = raw_track # Store raw data with full album object - print(f"βœ… New best ListenBrainz match: {spotify_result.artists[0]} - {spotify_result.name} (confidence: {combined_confidence:.3f})") + matched_track = search_result + best_raw_track = raw_track # Store raw data with full album object (Spotify only) + print(f"βœ… New best ListenBrainz match: {search_result.artists[0]} - {search_result.name} (confidence: {combined_confidence:.3f})") except Exception as e: print(f"❌ Error processing ListenBrainz search result: {e}") @@ -15665,59 +15841,76 @@ def _run_listenbrainz_discovery_worker(playlist_mbid): print(f"❌ Error in ListenBrainz search for query '{search_query}': {e}") continue - if spotify_track: - print(f"βœ… Strategy 1 ListenBrainz match: {spotify_track.artists[0]} - {spotify_track.name} (confidence: {best_confidence:.3f})") + if matched_track: + print(f"βœ… Strategy 1 ListenBrainz match: {matched_track.artists[0]} - {matched_track.name} (confidence: {best_confidence:.3f})") # Strategy 2: Swapped search (if first failed) - keep simple for fallback - if not spotify_track: + if not matched_track: print("πŸ”„ ListenBrainz Strategy 2: Trying swapped search (artist/title reversed)") query = f"artist:{cleaned_title} track:{cleaned_artist}" - spotify_results = spotify_client.search_tracks(query, limit=3) - if spotify_results: - spotify_track = spotify_results[0] - print(f"βœ… Strategy 2 ListenBrainz match (swapped): {spotify_track.artists[0]} - {spotify_track.name}") + if use_spotify: + fallback_results = spotify_client.search_tracks(query, limit=3) + else: + fallback_results = itunes_client.search_tracks(query, limit=3) + if fallback_results: + matched_track = fallback_results[0] + print(f"βœ… Strategy 2 ListenBrainz match (swapped): {matched_track.artists[0]} - {matched_track.name}") # Strategy 3: Album-based search (if still failed and we have album name) - if not spotify_track and album_name: + if not matched_track and album_name: print(f"πŸ”„ ListenBrainz Strategy 3: Trying album-based search: '{cleaned_artist} {album_name} {cleaned_title}'") query = f"artist:{cleaned_artist} album:{album_name} track:{cleaned_title}" - spotify_results = spotify_client.search_tracks(query, limit=3) - if spotify_results: - spotify_track = spotify_results[0] - print(f"βœ… Strategy 3 ListenBrainz match (album): {spotify_track.artists[0]} - {spotify_track.name}") + if use_spotify: + fallback_results = spotify_client.search_tracks(query, limit=3) + else: + fallback_results = itunes_client.search_tracks(query, limit=3) + if fallback_results: + matched_track = fallback_results[0] + print(f"βœ… Strategy 3 ListenBrainz match (album): {matched_track.artists[0]} - {matched_track.name}") # Create result entry result = { 'index': i, 'lb_track': cleaned_title, 'lb_artist': cleaned_artist, - 'status': 'βœ… Found' if spotify_track else '❌ Not Found', - 'status_class': 'found' if spotify_track else 'not-found', - 'spotify_track': spotify_track.name if spotify_track else '', - 'spotify_artist': spotify_track.artists[0] if spotify_track else '', - 'spotify_album': spotify_track.album if spotify_track else '', - 'duration': f"{duration_ms // 60000}:{(duration_ms % 60000) // 1000:02d}" if duration_ms else '0:00' + 'status': 'βœ… Found' if matched_track else '❌ Not Found', + 'status_class': 'found' if matched_track else 'not-found', + 'spotify_track': matched_track.name if matched_track else '', + 'spotify_artist': matched_track.artists[0] if matched_track else '', + 'spotify_album': matched_track.album if matched_track else '', + 'duration': f"{duration_ms // 60000}:{(duration_ms % 60000) // 1000:02d}" if duration_ms else '0:00', + 'discovery_source': discovery_source } - if spotify_track: - state['spotify_matches'] += 1 - # Use full album object from raw Spotify data if available - album_data = best_raw_track.get('album', {}) if best_raw_track else {} - if not album_data: - # Fallback to string album name - album_data = {'name': spotify_track.album, 'album_type': 'album', 'images': []} + if matched_track: + state['spotify_matches'] += 1 # Keep key name for compatibility - result['spotify_data'] = { - 'id': spotify_track.id, - 'name': spotify_track.name, - 'artists': spotify_track.artists, - 'album': album_data, # Full album object with images - 'duration_ms': spotify_track.duration_ms + # Build album data based on provider + if use_spotify and best_raw_track: + album_data = best_raw_track.get('album', {}) + else: + # For iTunes or when raw data unavailable + album_data = { + 'name': matched_track.album, + 'album_type': 'album', + 'images': [{'url': matched_track.image_url}] if hasattr(matched_track, 'image_url') and matched_track.image_url else [] + } + + # Store track data with source info + result['matched_data'] = { + 'id': matched_track.id, + 'name': matched_track.name, + 'artists': matched_track.artists, + 'album': album_data, + 'duration_ms': matched_track.duration_ms, + 'source': discovery_source } + # Keep spotify_data for backward compatibility + result['spotify_data'] = result['matched_data'] state['discovery_results'].append(result) - print(f" {'βœ…' if spotify_track else '❌'} Track {i+1}/{len(tracks)}: {result['status']}") + print(f" {'βœ…' if matched_track else '❌'} Track {i+1}/{len(tracks)}: {result['status']}") except Exception as e: print(f"❌ Error processing track {i}: {e}") @@ -15742,9 +15935,10 @@ def _run_listenbrainz_discovery_worker(playlist_mbid): # Add activity for discovery completion playlist_name = playlist['name'] - add_activity_item("βœ…", "ListenBrainz Discovery Complete", f"'{playlist_name}' - {state['spotify_matches']}/{len(tracks)} tracks found", "Now") + source_label = 'Spotify' if use_spotify else 'iTunes' + add_activity_item("βœ…", f"ListenBrainz Discovery Complete ({source_label})", f"'{playlist_name}' - {state['spotify_matches']}/{len(tracks)} tracks found", "Now") - print(f"βœ… ListenBrainz discovery complete: {state['spotify_matches']}/{len(tracks)} tracks matched") + print(f"βœ… ListenBrainz discovery complete ({discovery_source}): {state['spotify_matches']}/{len(tracks)} tracks matched") except Exception as e: print(f"❌ Error in ListenBrainz discovery worker: {e}") @@ -21483,21 +21677,28 @@ def clean_beatport_text(text): return text def _run_beatport_discovery_worker(url_hash): - """Background worker for Beatport Spotify discovery process""" + """Background worker for Beatport discovery process (Spotify preferred, iTunes fallback)""" try: state = beatport_chart_states[url_hash] chart = state['chart'] tracks = chart['tracks'] - print(f"πŸ” Starting Spotify discovery for {len(tracks)} Beatport tracks...") + # Determine which provider to use + use_spotify = spotify_client and spotify_client.is_spotify_authenticated() + discovery_source = 'spotify' if use_spotify else 'itunes' - if not spotify_client or not spotify_client.is_authenticated(): - print("❌ Spotify client not authenticated") - state['status'] = 'error' - state['phase'] = 'fresh' - return + # Initialize iTunes client if needed + itunes_client_instance = None + if not use_spotify: + from core.itunes_client import iTunesClient + itunes_client_instance = iTunesClient() - # Process each track for Spotify discovery + print(f"πŸ” Starting {discovery_source.upper()} discovery for {len(tracks)} Beatport tracks...") + + # Store discovery source in state for frontend + state['discovery_source'] = discovery_source + + # Process each track for discovery for i, track in enumerate(tracks): try: # Update progress @@ -21516,10 +21717,10 @@ def _run_beatport_discovery_worker(url_hash): else: track_artist = clean_beatport_text(str(track_artists)) - print(f"πŸ” Searching Spotify for: '{track_artist}' - '{track_title}'") + print(f"πŸ” Searching {discovery_source.upper()} for: '{track_artist}' - '{track_title}'") # Use matching engine for sophisticated track matching (like other discovery processes) - spotify_track = None + found_track = None # Generate search queries using matching engine (with fallback) try: @@ -21548,84 +21749,134 @@ def _run_beatport_discovery_worker(url_hash): for query_idx, search_query in enumerate(search_queries): try: - print(f"πŸ” Query {query_idx + 1}/{len(search_queries)}: {search_query}") + print(f"πŸ” Query {query_idx + 1}/{len(search_queries)}: {search_query} ({discovery_source.upper()})") - # Get raw Spotify API response to access full album object with images - raw_results = spotify_client.sp.search(q=search_query, type='track', limit=10) - if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']: - continue + if use_spotify: + # SPOTIFY PATH: Get raw Spotify API response to access full album object with images + raw_results = spotify_client.sp.search(q=search_query, type='track', limit=10) + if not raw_results or 'tracks' not in raw_results or not raw_results['tracks']['items']: + continue - search_results = spotify_client.search_tracks(search_query, limit=10) + search_results = spotify_client.search_tracks(search_query, limit=10) - if not search_results: - continue + if not search_results: + continue - # Use matching engine to find the best match from search results - for result_idx, result in enumerate(search_results): - raw_track = raw_results['tracks']['items'][result_idx] if result_idx < len(raw_results['tracks']['items']) else None - try: - # Calculate confidence using matching engine's similarity scoring (with fallback) + # Use matching engine to find the best match from search results + for result_idx, result in enumerate(search_results): + raw_track = raw_results['tracks']['items'][result_idx] if result_idx < len(raw_results['tracks']['items']) else None try: - artist_confidence = 0.0 - if result.artists: - # Get best artist match confidence - result_artist_names = [artist for artist in result.artists] - for result_artist in result_artist_names: + # Calculate confidence using matching engine's similarity scoring (with fallback) + try: + artist_confidence = 0.0 + if result.artists: + # Get best artist match confidence + result_artist_names = [artist for artist in result.artists] + for result_artist in result_artist_names: + artist_sim = matching_engine.similarity_score( + matching_engine.normalize_string(track_artist), + matching_engine.normalize_string(result_artist) + ) + artist_confidence = max(artist_confidence, artist_sim) + + # Calculate title confidence + title_confidence = matching_engine.similarity_score( + matching_engine.normalize_string(track_title), + matching_engine.normalize_string(result.name) + ) + + # Combined confidence (more balanced to avoid bad matches from same artist) + combined_confidence = (artist_confidence * 0.4 + title_confidence * 0.6) + except Exception as e: + print(f"⚠️ Matching engine scoring failed for Beatport, using basic matching: {e}") + # Fallback to simple string matching + artist_match = any(track_artist.lower() in artist.lower() for artist in result.artists) if result.artists else False + title_match = track_title.lower() in result.name.lower() or result.name.lower() in track_title.lower() + combined_confidence = 0.8 if (artist_match and title_match) else 0.4 if (artist_match or title_match) else 0.1 + + print(f"πŸ” Match candidate: '{result.artists[0]}' - '{result.name}'") + print(f" Artist confidence: {artist_confidence:.3f} ('{track_artist}' vs '{result.artists[0]}')") + print(f" Title confidence: {title_confidence:.3f} ('{track_title}' vs '{result.name}')") + print(f" Combined confidence: {combined_confidence:.3f} (threshold: {min_confidence})") + + # Additional check for core title similarity (excluding version keywords) + def remove_version_keywords(title): + keywords = ['extended mix', 'radio mix', 'club mix', 'remix', 'extended', 'version', 'mix', 'original'] + clean_title = title.lower() + for keyword in keywords: + clean_title = clean_title.replace(keyword, '').strip(' -()[]') + return clean_title.strip() + + core_title1 = remove_version_keywords(track_title) + core_title2 = remove_version_keywords(result.name) + core_title_confidence = matching_engine.similarity_score(core_title1, core_title2) + + print(f" Core title confidence: {core_title_confidence:.3f} ('{core_title1}' vs '{core_title2}')") + + # Update best match if this is better AND meets all similarity requirements + min_title_confidence = 0.5 # Require at least 50% title similarity + min_core_title_confidence = 0.4 # Require at least 40% core title similarity + if (combined_confidence > best_confidence and + combined_confidence >= min_confidence and + title_confidence >= min_title_confidence and + core_title_confidence >= min_core_title_confidence): + best_confidence = combined_confidence + best_match = result + best_raw_track = raw_track # Store raw data with full album object + print(f"βœ… New best match: {result.artists[0]} - {result.name} (confidence: {combined_confidence:.3f})") + + except Exception as e: + print(f"❌ Error processing search result: {e}") + continue + + else: + # ITUNES PATH: Search using iTunes client + simple_query = f"{track_artist} {track_title}" + itunes_results = itunes_client_instance.search_tracks(simple_query, limit=10) + + if not itunes_results: + continue + + # Score each iTunes result + for result in itunes_results: + try: + # Calculate confidence using matching engine + try: + artist_confidence = 0.0 + result_artist = result.artist if hasattr(result, 'artist') else result.get('artist', '') + if result_artist: artist_sim = matching_engine.similarity_score( matching_engine.normalize_string(track_artist), matching_engine.normalize_string(result_artist) ) - artist_confidence = max(artist_confidence, artist_sim) + artist_confidence = artist_sim - # Calculate title confidence - title_confidence = matching_engine.similarity_score( - matching_engine.normalize_string(track_title), - matching_engine.normalize_string(result.name) - ) + # Calculate title confidence + result_name = result.name if hasattr(result, 'name') else result.get('name', '') + title_confidence = matching_engine.similarity_score( + matching_engine.normalize_string(track_title), + matching_engine.normalize_string(result_name) + ) + + combined_confidence = (artist_confidence * 0.4 + title_confidence * 0.6) + except Exception as e: + print(f"⚠️ Matching engine scoring failed for iTunes Beatport, using first match: {e}") + combined_confidence = 1.0 + best_match = result + break + + result_artist_display = result.artist if hasattr(result, 'artist') else result.get('artist', 'Unknown') + result_name_display = result.name if hasattr(result, 'name') else result.get('name', 'Unknown') + print(f"πŸ” iTunes Beatport candidate: '{result_artist_display}' - '{result_name_display}' (confidence: {combined_confidence:.3f})") + + if combined_confidence > best_confidence and combined_confidence >= min_confidence: + best_confidence = combined_confidence + best_match = result + print(f"βœ… New best iTunes Beatport match: {result_artist_display} - {result_name_display} (confidence: {combined_confidence:.3f})") - # Combined confidence (more balanced to avoid bad matches from same artist) - combined_confidence = (artist_confidence * 0.4 + title_confidence * 0.6) except Exception as e: - print(f"⚠️ Matching engine scoring failed for Beatport, using basic matching: {e}") - # Fallback to simple string matching - artist_match = any(track_artist.lower() in artist.lower() for artist in result.artists) if result.artists else False - title_match = track_title.lower() in result.name.lower() or result.name.lower() in track_title.lower() - combined_confidence = 0.8 if (artist_match and title_match) else 0.4 if (artist_match or title_match) else 0.1 - - print(f"πŸ” Match candidate: '{result.artists[0]}' - '{result.name}'") - print(f" Artist confidence: {artist_confidence:.3f} ('{track_artist}' vs '{result.artists[0]}')") - print(f" Title confidence: {title_confidence:.3f} ('{track_title}' vs '{result.name}')") - print(f" Combined confidence: {combined_confidence:.3f} (threshold: {min_confidence})") - - # Additional check for core title similarity (excluding version keywords) - def remove_version_keywords(title): - keywords = ['extended mix', 'radio mix', 'club mix', 'remix', 'extended', 'version', 'mix', 'original'] - clean_title = title.lower() - for keyword in keywords: - clean_title = clean_title.replace(keyword, '').strip(' -()[]') - return clean_title.strip() - - core_title1 = remove_version_keywords(track_title) - core_title2 = remove_version_keywords(result.name) - core_title_confidence = matching_engine.similarity_score(core_title1, core_title2) - - print(f" Core title confidence: {core_title_confidence:.3f} ('{core_title1}' vs '{core_title2}')") - - # Update best match if this is better AND meets all similarity requirements - min_title_confidence = 0.5 # Require at least 50% title similarity - min_core_title_confidence = 0.4 # Require at least 40% core title similarity - if (combined_confidence > best_confidence and - combined_confidence >= min_confidence and - title_confidence >= min_title_confidence and - core_title_confidence >= min_core_title_confidence): - best_confidence = combined_confidence - best_match = result - best_raw_track = raw_track # Store raw data with full album object - print(f"βœ… New best match: {result.artists[0]} - {result.name} (confidence: {combined_confidence:.3f})") - - except Exception as e: - print(f"❌ Error processing search result: {e}") - continue + print(f"❌ Error processing iTunes Beatport search result: {e}") + continue # If we found a very high confidence match, stop searching if best_confidence >= 0.9: @@ -21633,12 +21884,17 @@ def _run_beatport_discovery_worker(url_hash): break except Exception as e: - print(f"❌ Error in Spotify search for query '{search_query}': {e}") + print(f"❌ Error in {discovery_source.upper()} search for query '{search_query}': {e}") continue - spotify_track = best_match - if spotify_track: - print(f"βœ… Final match selected: {spotify_track.artists[0]} - {spotify_track.name} (confidence: {best_confidence:.3f})") + found_track = best_match + if found_track: + if use_spotify: + print(f"βœ… Final Spotify match selected: {found_track.artists[0]} - {found_track.name} (confidence: {best_confidence:.3f})") + else: + result_artist = found_track.artist if hasattr(found_track, 'artist') else found_track.get('artist', 'Unknown') + result_name = found_track.name if hasattr(found_track, 'name') else found_track.get('name', 'Unknown') + print(f"βœ… Final iTunes match selected: {result_artist} - {result_name} (confidence: {best_confidence:.3f})") else: print(f"❌ No suitable match found (best confidence was {best_confidence:.3f}, required {min_confidence:.3f})") @@ -21649,41 +21905,70 @@ def _run_beatport_discovery_worker(url_hash): 'title': track_title, 'artist': track_artist }, - 'status': 'found' if spotify_track else 'not_found', - 'status_class': 'found' if spotify_track else 'not-found' # Add status class for CSS styling + 'status': 'found' if found_track else 'not_found', + 'status_class': 'found' if found_track else 'not-found', # Add status class for CSS styling + 'discovery_source': discovery_source } - if spotify_track: - # Debug: show available attributes - print(f"πŸ” Spotify track attributes: {dir(spotify_track)}") + if found_track: + if use_spotify: + # SPOTIFY result formatting + # Debug: show available attributes + print(f"πŸ” Spotify track attributes: {dir(found_track)}") - # Format artists correctly for frontend compatibility - formatted_artists = [] - if isinstance(spotify_track.artists, list): - # If it's already a list of strings, convert to objects with 'name' property - for artist in spotify_track.artists: - if isinstance(artist, str): - formatted_artists.append({'name': artist}) - else: - # If it's already an object, use as-is - formatted_artists.append(artist) + # Format artists correctly for frontend compatibility + formatted_artists = [] + if isinstance(found_track.artists, list): + # If it's already a list of strings, convert to objects with 'name' property + for artist in found_track.artists: + if isinstance(artist, str): + formatted_artists.append({'name': artist}) + else: + # If it's already an object, use as-is + formatted_artists.append(artist) + else: + # Single artist case + formatted_artists = [{'name': str(found_track.artists)}] + + # Use full album object from raw Spotify data if available + album_data = best_raw_track.get('album', {}) if best_raw_track else {} + if not album_data: + # Fallback to string album name + album_data = {'name': found_track.album, 'album_type': 'album', 'images': []} + + result_entry['spotify_data'] = { + 'name': found_track.name, + 'artists': formatted_artists, # Now formatted as list of objects with 'name' property + 'album': album_data, # Full album object with images + 'id': found_track.id, + 'source': 'spotify' + } else: - # Single artist case - formatted_artists = [{'name': str(spotify_track.artists)}] + # ITUNES result formatting + result_artist = found_track.artist if hasattr(found_track, 'artist') else found_track.get('artist', 'Unknown') + result_name = found_track.name if hasattr(found_track, 'name') else found_track.get('name', 'Unknown') + album_name = found_track.album if hasattr(found_track, 'album') else found_track.get('album', 'Unknown Album') + artwork_url = found_track.artwork_url if hasattr(found_track, 'artwork_url') else found_track.get('artwork_url', '') + track_id = found_track.id if hasattr(found_track, 'id') else found_track.get('id', '') - # Use full album object from raw Spotify data if available - album_data = best_raw_track.get('album', {}) if best_raw_track else {} - if not album_data: - # Fallback to string album name - album_data = {'name': spotify_track.album, 'album_type': 'album', 'images': []} + # Format artists as list of objects for frontend compatibility + formatted_artists = [{'name': result_artist}] + + # Build album data with artwork + album_data = { + 'name': album_name, + 'album_type': 'album', + 'images': [{'url': artwork_url, 'height': 300, 'width': 300}] if artwork_url else [] + } + + result_entry['spotify_data'] = { # Use same key for frontend compatibility + 'name': result_name, + 'artists': formatted_artists, + 'album': album_data, + 'id': track_id, + 'source': 'itunes' + } - result_entry['spotify_data'] = { - 'name': spotify_track.name, - 'artists': formatted_artists, # Now formatted as list of objects with 'name' property - 'album': album_data, # Full album object with images - 'id': spotify_track.id - # Remove uri for now since it's causing errors - } state['spotify_matches'] += 1 state['discovery_results'].append(result_entry) @@ -21702,7 +21987,8 @@ def _run_beatport_discovery_worker(url_hash): }, 'status': 'error', 'status_class': 'error', # Add status class for CSS styling - 'error': str(e) + 'error': str(e), + 'discovery_source': discovery_source }) # Mark discovery as complete @@ -21712,10 +21998,11 @@ def _run_beatport_discovery_worker(url_hash): # Add activity for completion chart_name = chart.get('name', 'Unknown Chart') - add_activity_item("βœ…", "Beatport Discovery Complete", + source_label = discovery_source.upper() + add_activity_item("βœ…", f"Beatport Discovery Complete ({source_label})", f"'{chart_name}' - {state['spotify_matches']}/{len(tracks)} tracks found", "Now") - print(f"βœ… Beatport discovery complete: {state['spotify_matches']}/{len(tracks)} tracks found") + print(f"βœ… Beatport discovery complete ({source_label}): {state['spotify_matches']}/{len(tracks)} tracks found") except Exception as e: print(f"❌ Error in Beatport discovery worker: {e}") From 2170ffa99e98e4a7b5dba01b26c6f90ce292b443 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sat, 24 Jan 2026 14:25:27 -0800 Subject: [PATCH 10/20] Fix discovery modal fix button for iTunes source and ListenBrainz playlists - Fix platform detection to include is_listenbrainz_playlist check when generating fix buttons - Update openDiscoveryFixModal to check both listenbrainzPlaylistStates and youtubePlaylistStates - Update searchDiscoveryFix to detect discovery_source and use appropriate search API - Add /api/itunes/search_tracks endpoint for manual track search when using iTunes source - Update selectDiscoveryFixTrack state lookup to include ListenBrainz Fix button now works correctly for all platforms (YouTube, ListenBrainz, Tidal, Beatport) with both Spotify and iTunes discovery sources. --- web_server.py | 34 ++++++++++++++++++++++++++++++++++ webui/static/script.js | 32 +++++++++++++++++++++++--------- 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/web_server.py b/web_server.py index c18871be..27858c83 100644 --- a/web_server.py +++ b/web_server.py @@ -14293,6 +14293,40 @@ def search_spotify_tracks(): return jsonify({"error": str(e)}), 500 +@app.route('/api/itunes/search_tracks', methods=['GET']) +def search_itunes_tracks(): + """Search for tracks on iTunes - used by discovery fix modal when iTunes is the source""" + try: + from core.itunes_client import iTunesClient + + query = request.args.get('query', '').strip() + limit = int(request.args.get('limit', 20)) + + if not query: + return jsonify({"error": "Query parameter is required"}), 400 + + # Search using iTunes client + itunes_client = iTunesClient() + tracks = itunes_client.search_tracks(query, limit=limit) + + # Convert tracks to dict format matching Spotify structure for frontend compatibility + tracks_dict = [{ + 'id': t.id, + 'name': t.name, + 'artists': t.artists, # Already a list + 'album': t.album, + 'duration_ms': t.duration_ms, + 'image_url': t.image_url, + 'source': 'itunes' + } for t in tracks] + + return jsonify({'tracks': tracks_dict}) + + except Exception as e: + print(f"❌ Error searching iTunes tracks: {e}") + return jsonify({"error": str(e)}), 500 + + # =================================================================== # TIDAL PLAYLIST API ENDPOINTS # =================================================================== diff --git a/webui/static/script.js b/webui/static/script.js index 88969d05..98cd6e44 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -9946,7 +9946,8 @@ function openDiscoveryFixModal(platform, identifier, trackIndex) { // Note: Beatport, Tidal, and ListenBrainz have their own states, but reuse YouTube modal infrastructure let state, result; if (platform === 'youtube') { - state = youtubePlaylistStates[identifier]; + // Check both states - ListenBrainz also uses YouTube modal infrastructure + state = listenbrainzPlaylistStates[identifier] || youtubePlaylistStates[identifier]; } else if (platform === 'tidal') { state = youtubePlaylistStates[identifier]; // Tidal uses YouTube state infrastructure } else if (platform === 'beatport') { @@ -10099,15 +10100,23 @@ async function searchDiscoveryFix() { return; } + // Determine discovery source from state + const identifier = currentDiscoveryFix.identifier; + const state = listenbrainzPlaylistStates[identifier] || youtubePlaylistStates[identifier]; + const discoverySource = state?.discovery_source || state?.discoverySource || 'spotify'; + const useItunes = discoverySource === 'itunes'; + const resultsContainer = fixModalOverlay.querySelector('#fix-modal-results'); - resultsContainer.innerHTML = '
πŸ” Searching Spotify...
'; + const sourceLabel = useItunes ? 'iTunes' : 'Spotify'; + resultsContainer.innerHTML = `
πŸ” Searching ${sourceLabel}...
`; try { // Build search query const query = `${artistInput} ${trackInput}`.trim(); - // Call Spotify search API - const response = await fetch(`/api/spotify/search_tracks?query=${encodeURIComponent(query)}&limit=20`); + // Call appropriate search API based on discovery source + const searchEndpoint = useItunes ? '/api/itunes/search_tracks' : '/api/spotify/search_tracks'; + const response = await fetch(`${searchEndpoint}?query=${encodeURIComponent(query)}&limit=20`); const data = await response.json(); if (data.error) { @@ -10120,7 +10129,7 @@ async function searchDiscoveryFix() { return; } - // Render results + // Render results (same format for both Spotify and iTunes) renderDiscoveryFixResults(data.tracks, fixModalOverlay); } catch (error) { @@ -10216,13 +10225,16 @@ async function selectDiscoveryFixTrack(track) { // Update frontend state // Note: Beatport and Tidal reuse youtubePlaylistStates for discovery results + // ListenBrainz uses its own state but may also be accessed via YouTube let state; if (platform === 'youtube') { - state = youtubePlaylistStates[identifier]; + state = listenbrainzPlaylistStates[identifier] || youtubePlaylistStates[identifier]; } else if (platform === 'tidal') { state = youtubePlaylistStates[identifier]; } else if (platform === 'beatport') { state = youtubePlaylistStates[identifier]; + } else if (platform === 'listenbrainz') { + state = listenbrainzPlaylistStates[identifier]; } // Support both camelCase and snake_case @@ -18602,15 +18614,17 @@ function updateYouTubeDiscoveryModal(urlHash, status) { // Update actions cell with appropriate button if (actionsCell) { - const state = youtubePlaylistStates[urlHash]; - const platform = state?.is_tidal_playlist ? 'tidal' : (state?.is_beatport_playlist ? 'beatport' : 'youtube'); + const state = listenbrainzPlaylistStates[urlHash] || youtubePlaylistStates[urlHash]; + const platform = state?.is_listenbrainz_playlist ? 'listenbrainz' : + (state?.is_tidal_playlist ? 'tidal' : + (state?.is_beatport_playlist ? 'beatport' : 'youtube')); actionsCell.innerHTML = generateDiscoveryActionButton(result, urlHash, platform); } }); // Update action buttons if discovery is complete (progress = 100%) if (status.progress >= 100) { - const state = youtubePlaylistStates[urlHash]; + const state = listenbrainzPlaylistStates[urlHash] || youtubePlaylistStates[urlHash]; if (state && state.phase === 'discovered') { const actionButtonsContainer = document.querySelector(`#youtube-discovery-modal-${urlHash} .modal-footer-left`); if (actionButtonsContainer) { From 580d28cfb98b53b0bf6c6a5edcd9c2335378efe2 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sat, 24 Jan 2026 14:35:54 -0800 Subject: [PATCH 11/20] Update web_server.py --- web_server.py | 52 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/web_server.py b/web_server.py index 27858c83..b37fb097 100644 --- a/web_server.py +++ b/web_server.py @@ -15025,12 +15025,15 @@ def _search_spotify_for_tidal_track(tidal_track, use_spotify=True, itunes_client continue # Score each iTunes result + # Note: iTunes returns Track dataclass objects with 'artists' (list), not 'artist' for result in itunes_results: try: # Calculate confidence using matching engine try: artist_confidence = 0.0 - result_artist = result.artist if hasattr(result, 'artist') else result.get('artist', '') + # iTunes Track has 'artists' as a list + result_artists = result.artists if hasattr(result, 'artists') else [] + result_artist = result_artists[0] if result_artists else '' if result_artist: artist_sim = matching_engine.similarity_score( matching_engine.normalize_string(artist_name), @@ -15039,7 +15042,7 @@ def _search_spotify_for_tidal_track(tidal_track, use_spotify=True, itunes_client artist_confidence = artist_sim # Calculate title confidence - result_name = result.name if hasattr(result, 'name') else result.get('name', '') + result_name = result.name if hasattr(result, 'name') else '' title_confidence = matching_engine.similarity_score( matching_engine.normalize_string(track_name), matching_engine.normalize_string(result_name) @@ -15052,8 +15055,8 @@ def _search_spotify_for_tidal_track(tidal_track, use_spotify=True, itunes_client best_match = result break - result_artist_display = result.artist if hasattr(result, 'artist') else result.get('artist', 'Unknown') - result_name_display = result.name if hasattr(result, 'name') else result.get('name', 'Unknown') + result_artist_display = result_artists[0] if result_artists else 'Unknown' + result_name_display = result.name if hasattr(result, 'name') else 'Unknown' print(f"πŸ” iTunes Tidal candidate: '{result_artist_display}' - '{result_name_display}' (confidence: {combined_confidence:.3f})") if combined_confidence > best_confidence and combined_confidence >= min_confidence: @@ -15080,15 +15083,17 @@ def _search_spotify_for_tidal_track(tidal_track, use_spotify=True, itunes_client return (best_match, best_match_raw) # Return both Track object and raw data else: # For iTunes, return a dict with normalized data - result_artist = best_match.artist if hasattr(best_match, 'artist') else best_match.get('artist', 'Unknown') - result_name = best_match.name if hasattr(best_match, 'name') else best_match.get('name', 'Unknown') + # Note: iTunes Track dataclass has 'artists' (list) and 'image_url', not 'artist' and 'artwork_url' + result_artists = best_match.artists if hasattr(best_match, 'artists') else [] + result_artist = result_artists[0] if result_artists else 'Unknown' + result_name = best_match.name if hasattr(best_match, 'name') else 'Unknown' print(f"βœ… Final Tidal iTunes match: {result_artist} - {result_name} (confidence: {best_confidence:.3f})") # Build iTunes result dict with album info - album_name = best_match.album if hasattr(best_match, 'album') else best_match.get('album', 'Unknown Album') - artwork_url = best_match.artwork_url if hasattr(best_match, 'artwork_url') else best_match.get('artwork_url', '') - track_id = best_match.id if hasattr(best_match, 'id') else best_match.get('id', '') - duration_ms = best_match.duration_ms if hasattr(best_match, 'duration_ms') else best_match.get('duration_ms', 0) + album_name = best_match.album if hasattr(best_match, 'album') else 'Unknown Album' + image_url = best_match.image_url if hasattr(best_match, 'image_url') else '' + track_id = best_match.id if hasattr(best_match, 'id') else '' + duration_ms = best_match.duration_ms if hasattr(best_match, 'duration_ms') else 0 return { 'id': track_id, @@ -15097,7 +15102,7 @@ def _search_spotify_for_tidal_track(tidal_track, use_spotify=True, itunes_client 'album': { 'name': album_name, 'album_type': 'album', - 'images': [{'url': artwork_url, 'height': 300, 'width': 300}] if artwork_url else [] + 'images': [{'url': image_url, 'height': 300, 'width': 300}] if image_url else [] }, 'duration_ms': duration_ms, 'source': 'itunes' @@ -21872,12 +21877,15 @@ def _run_beatport_discovery_worker(url_hash): continue # Score each iTunes result + # Note: iTunes returns Track dataclass objects with 'artists' (list), not 'artist' for result in itunes_results: try: # Calculate confidence using matching engine try: artist_confidence = 0.0 - result_artist = result.artist if hasattr(result, 'artist') else result.get('artist', '') + # iTunes Track has 'artists' as a list + result_artists = result.artists if hasattr(result, 'artists') else [] + result_artist = result_artists[0] if result_artists else '' if result_artist: artist_sim = matching_engine.similarity_score( matching_engine.normalize_string(track_artist), @@ -21886,7 +21894,7 @@ def _run_beatport_discovery_worker(url_hash): artist_confidence = artist_sim # Calculate title confidence - result_name = result.name if hasattr(result, 'name') else result.get('name', '') + result_name = result.name if hasattr(result, 'name') else '' title_confidence = matching_engine.similarity_score( matching_engine.normalize_string(track_title), matching_engine.normalize_string(result_name) @@ -21899,8 +21907,8 @@ def _run_beatport_discovery_worker(url_hash): best_match = result break - result_artist_display = result.artist if hasattr(result, 'artist') else result.get('artist', 'Unknown') - result_name_display = result.name if hasattr(result, 'name') else result.get('name', 'Unknown') + result_artist_display = result_artists[0] if result_artists else 'Unknown' + result_name_display = result.name if hasattr(result, 'name') else 'Unknown' print(f"πŸ” iTunes Beatport candidate: '{result_artist_display}' - '{result_name_display}' (confidence: {combined_confidence:.3f})") if combined_confidence > best_confidence and combined_confidence >= min_confidence: @@ -21979,11 +21987,13 @@ def _run_beatport_discovery_worker(url_hash): } else: # ITUNES result formatting - result_artist = found_track.artist if hasattr(found_track, 'artist') else found_track.get('artist', 'Unknown') - result_name = found_track.name if hasattr(found_track, 'name') else found_track.get('name', 'Unknown') - album_name = found_track.album if hasattr(found_track, 'album') else found_track.get('album', 'Unknown Album') - artwork_url = found_track.artwork_url if hasattr(found_track, 'artwork_url') else found_track.get('artwork_url', '') - track_id = found_track.id if hasattr(found_track, 'id') else found_track.get('id', '') + # Note: iTunes Track dataclass has 'artists' (list) and 'image_url', not 'artist' and 'artwork_url' + result_artists = found_track.artists if hasattr(found_track, 'artists') else [] + result_artist = result_artists[0] if result_artists else 'Unknown' + result_name = found_track.name if hasattr(found_track, 'name') else 'Unknown' + album_name = found_track.album if hasattr(found_track, 'album') else 'Unknown Album' + image_url = found_track.image_url if hasattr(found_track, 'image_url') else '' + track_id = found_track.id if hasattr(found_track, 'id') else '' # Format artists as list of objects for frontend compatibility formatted_artists = [{'name': result_artist}] @@ -21992,7 +22002,7 @@ def _run_beatport_discovery_worker(url_hash): album_data = { 'name': album_name, 'album_type': 'album', - 'images': [{'url': artwork_url, 'height': 300, 'width': 300}] if artwork_url else [] + 'images': [{'url': image_url, 'height': 300, 'width': 300}] if image_url else [] } result_entry['spotify_data'] = { # Use same key for frontend compatibility From b0e34c6942d0817568c5e0ca4d374fa357e77960 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sat, 24 Jan 2026 14:53:43 -0800 Subject: [PATCH 12/20] Update web_server.py --- web_server.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web_server.py b/web_server.py index b37fb097..350abaad 100644 --- a/web_server.py +++ b/web_server.py @@ -21934,9 +21934,11 @@ def _run_beatport_discovery_worker(url_hash): if use_spotify: print(f"βœ… Final Spotify match selected: {found_track.artists[0]} - {found_track.name} (confidence: {best_confidence:.3f})") else: - result_artist = found_track.artist if hasattr(found_track, 'artist') else found_track.get('artist', 'Unknown') - result_name = found_track.name if hasattr(found_track, 'name') else found_track.get('name', 'Unknown') - print(f"βœ… Final iTunes match selected: {result_artist} - {result_name} (confidence: {best_confidence:.3f})") + # iTunes Track has 'artists' (list), not 'artist' + found_artists = found_track.artists if hasattr(found_track, 'artists') else [] + found_artist = found_artists[0] if found_artists else 'Unknown' + found_name = found_track.name if hasattr(found_track, 'name') else 'Unknown' + print(f"βœ… Final iTunes match selected: {found_artist} - {found_name} (confidence: {best_confidence:.3f})") else: print(f"❌ No suitable match found (best confidence was {best_confidence:.3f}, required {min_confidence:.3f})") From 3cb88669e3e7d6e7d5aa6278768aca93ab2f4a47 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sat, 24 Jan 2026 21:48:17 -0800 Subject: [PATCH 13/20] Fix iTunes-only Discover page not loading data - Add similar artists fetching to web UI scan loop - Add database migration for UNIQUE constraint on similar_artists table - Add source-agnostic /api/discover/album endpoint for iTunes support - Fix NOT NULL constraint on discovery_recent_albums blocking iTunes albums - Add fallback to watchlist artists when no similar artists exist - Add /api/discover/refresh and /api/discover/diagnose endpoints - Add retry logic with exponential backoff for iTunes API calls - Ensure cache_discovery_recent_albums runs even when pool population skips --- core/watchlist_scanner.py | 147 ++++++++++++-- database/music_database.py | 84 ++++++++ tools/diagnose_itunes_discover.py | 270 +++++++++++++++++++++++++ web_server.py | 320 +++++++++++++++++++++++++++++- webui/static/script.js | 21 +- 5 files changed, 808 insertions(+), 34 deletions(-) create mode 100644 tools/diagnose_itunes_discover.py diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index db7c9b17..367ebddf 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -21,9 +21,61 @@ logger = get_logger("watchlist_scanner") # Rate limiting constants for watchlist operations DELAY_BETWEEN_ARTISTS = 2.0 # 2 seconds between different artists -DELAY_BETWEEN_ALBUMS = 0.5 # 500ms between albums for same artist +DELAY_BETWEEN_ALBUMS = 0.5 # 500ms between albums for same artist DELAY_BETWEEN_API_BATCHES = 1.0 # 1 second between API batch operations +# iTunes API retry configuration +ITUNES_MAX_RETRIES = 3 +ITUNES_BASE_DELAY = 1.0 # Base delay in seconds for exponential backoff + + +def itunes_api_call_with_retry(func, *args, max_retries=ITUNES_MAX_RETRIES, **kwargs): + """ + Execute an iTunes API call with exponential backoff retry logic. + + Args: + func: The function to call + *args: Arguments to pass to the function + max_retries: Maximum number of retry attempts + **kwargs: Keyword arguments to pass to the function + + Returns: + The result of the function call, or None if all retries failed + """ + last_error = None + for attempt in range(max_retries): + try: + result = func(*args, **kwargs) + return result + except requests.exceptions.HTTPError as e: + # Handle rate limiting (429) and server errors (5xx) + if e.response is not None and e.response.status_code == 429: + delay = ITUNES_BASE_DELAY * (2 ** attempt) + logger.warning(f"[iTunes] Rate limited, retrying in {delay}s (attempt {attempt + 1}/{max_retries})") + time.sleep(delay) + last_error = e + elif e.response is not None and e.response.status_code >= 500: + delay = ITUNES_BASE_DELAY * (2 ** attempt) + logger.warning(f"[iTunes] Server error {e.response.status_code}, retrying in {delay}s (attempt {attempt + 1}/{max_retries})") + time.sleep(delay) + last_error = e + else: + raise # Don't retry on client errors (4xx except 429) + except requests.exceptions.RequestException as e: + # Retry on connection errors + delay = ITUNES_BASE_DELAY * (2 ** attempt) + logger.warning(f"[iTunes] Connection error, retrying in {delay}s (attempt {attempt + 1}/{max_retries}): {e}") + time.sleep(delay) + last_error = e + except Exception as e: + # Don't retry on other exceptions + raise + + if last_error: + logger.error(f"[iTunes] All {max_retries} retry attempts failed: {last_error}") + return None + + def clean_track_name_for_search(track_name): """ Intelligently cleans a track name for searching by removing noise while preserving important version information. @@ -1287,9 +1339,11 @@ class WatchlistScanner: except Exception as e: logger.debug(f"Spotify match failed for {artist_name_to_match}: {e}") - # Try to match on iTunes + # Try to match on iTunes (with retry for rate limiting) try: - itunes_results = itunes_client.search_artists(artist_name_to_match, limit=1) + itunes_results = itunes_api_call_with_retry( + itunes_client.search_artists, artist_name_to_match, limit=1 + ) if itunes_results and len(itunes_results) > 0: itunes_artist = itunes_results[0] # Skip if this is the searched artist @@ -1301,8 +1355,10 @@ class WatchlistScanner: # Use iTunes genres if we don't have Spotify genres if not artist_data['genres'] and hasattr(itunes_artist, 'genres'): artist_data['genres'] = itunes_artist.genres + else: + logger.info(f" [iTunes] No match found for: {artist_name_to_match}") except Exception as e: - logger.debug(f"iTunes match failed for {artist_name_to_match}: {e}") + logger.info(f" [iTunes] Match failed for {artist_name_to_match}: {e}") # Only add if we got at least one ID if artist_data['spotify_id'] or artist_data['itunes_id']: @@ -1314,7 +1370,11 @@ class WatchlistScanner: logger.debug(f"Error matching {artist_name_to_match}: {match_error}") continue - logger.info(f"Matched {len(matched_artists)} similar artists (Spotify + iTunes)") + # Log detailed matching statistics + itunes_matched = sum(1 for a in matched_artists if a.get('itunes_id')) + spotify_matched = sum(1 for a in matched_artists if a.get('spotify_id')) + both_matched = sum(1 for a in matched_artists if a.get('itunes_id') and a.get('spotify_id')) + logger.info(f"Matched {len(matched_artists)} similar artists - iTunes: {itunes_matched}, Spotify: {spotify_matched}, Both: {both_matched}") return matched_artists except requests.exceptions.RequestException as e: @@ -1436,9 +1496,15 @@ class WatchlistScanner: from datetime import datetime, timedelta import random - # Check if we should run (prevents over-polling) - if not self.database.should_populate_discovery_pool(hours_threshold=24): - logger.info("Discovery pool was populated recently (< 24 hours ago). Skipping.") + # Check if we should run discovery pool population (prevents over-polling) + skip_pool_population = not self.database.should_populate_discovery_pool(hours_threshold=24) + + if skip_pool_population: + logger.info("Discovery pool was populated recently (< 24 hours ago). Skipping pool population.") + logger.info("But still refreshing recent albums cache and curated playlists...") + # Still run these even when skipping main pool population + self.cache_discovery_recent_albums() + self.curate_discovery_playlists() return logger.info("Populating discovery pool from similar artists...") @@ -1461,7 +1527,11 @@ class WatchlistScanner: similar_artists = self.database.get_top_similar_artists(limit=top_artists_limit) if not similar_artists: - logger.info("No similar artists found to populate discovery pool") + logger.info("No similar artists found to populate discovery pool from similar artists") + logger.info("But still caching recent albums from watchlist artists and curating playlists...") + # Still run these even without similar artists - they use watchlist artists + self.cache_discovery_recent_albums() + self.curate_discovery_playlists() return logger.info(f"Processing {len(similar_artists)} top similar artists for discovery pool") @@ -2056,26 +2126,40 @@ class WatchlistScanner: logger.debug(f"Error processing album: {e}") return False + # Track resolution stats + itunes_resolved = 0 + itunes_failed_resolve = 0 + # Process watchlist artists for artist in watchlist_artists: # Always process iTunes (baseline) itunes_id = artist.itunes_artist_id if not itunes_id: - # Try to resolve iTunes ID on-the-fly + # Try to resolve iTunes ID on-the-fly (with retry for rate limiting) try: - results = itunes_client.search_artists(artist.artist_name, limit=1) + results = itunes_api_call_with_retry( + itunes_client.search_artists, artist.artist_name, limit=1 + ) if results and len(results) > 0: itunes_id = results[0].id - except: - pass + itunes_resolved += 1 + logger.debug(f"[iTunes] Resolved ID for {artist.artist_name}: {itunes_id}") + else: + itunes_failed_resolve += 1 + logger.info(f"[iTunes] No artist found for: {artist.artist_name}") + except Exception as e: + itunes_failed_resolve += 1 + logger.info(f"[iTunes] Failed to resolve {artist.artist_name}: {e}") if itunes_id: try: - albums = itunes_client.get_artist_albums(itunes_id, album_type='album,single', limit=20) + albums = itunes_api_call_with_retry( + itunes_client.get_artist_albums, itunes_id, album_type='album,single', limit=20 + ) for album in albums or []: process_album(album, artist.artist_name, artist.spotify_artist_id, itunes_id, 'itunes') except Exception as e: - logger.debug(f"Error fetching iTunes albums for {artist.artist_name}: {e}") + logger.info(f"[iTunes] Error fetching albums for {artist.artist_name}: {e}") # Process Spotify if authenticated if spotify_available and artist.spotify_artist_id: @@ -2097,23 +2181,33 @@ class WatchlistScanner: # Always process iTunes (baseline) itunes_id = artist.similar_artist_itunes_id if not itunes_id: - # Try to resolve iTunes ID on-the-fly + # Try to resolve iTunes ID on-the-fly (with retry for rate limiting) try: - results = itunes_client.search_artists(artist.similar_artist_name, limit=1) + results = itunes_api_call_with_retry( + itunes_client.search_artists, artist.similar_artist_name, limit=1 + ) if results and len(results) > 0: itunes_id = results[0].id # Cache for future self.database.update_similar_artist_itunes_id(artist.id, itunes_id) - except: - pass + itunes_resolved += 1 + logger.debug(f"[iTunes] Resolved ID for similar artist {artist.similar_artist_name}: {itunes_id}") + else: + itunes_failed_resolve += 1 + logger.info(f"[iTunes] No artist found for similar: {artist.similar_artist_name}") + except Exception as e: + itunes_failed_resolve += 1 + logger.info(f"[iTunes] Failed to resolve similar {artist.similar_artist_name}: {e}") if itunes_id: try: - albums = itunes_client.get_artist_albums(itunes_id, album_type='album,single', limit=20) + albums = itunes_api_call_with_retry( + itunes_client.get_artist_albums, itunes_id, album_type='album,single', limit=20 + ) for album in albums or []: process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, itunes_id, 'itunes') except Exception as e: - logger.debug(f"Error fetching iTunes albums for {artist.similar_artist_name}: {e}") + logger.info(f"[iTunes] Error fetching albums for similar {artist.similar_artist_name}: {e}") # Process Spotify if authenticated if spotify_available and artist.similar_artist_spotify_id: @@ -2132,6 +2226,7 @@ class WatchlistScanner: total_cached = cached_count['spotify'] + cached_count['itunes'] logger.info(f"Cached {total_cached} recent albums (Spotify: {cached_count['spotify']}, iTunes: {cached_count['itunes']}) from {albums_checked} albums checked") + logger.info(f"[iTunes] ID resolution stats: {itunes_resolved} resolved, {itunes_failed_resolve} failed") except Exception as e: logger.error(f"Error caching discovery recent albums: {e}") @@ -2171,6 +2266,9 @@ class WatchlistScanner: recent_albums = self.database.get_discovery_recent_albums(limit=50, source=source) release_radar_tracks = [] + if not recent_albums: + logger.warning(f"[{source.upper()}] No recent albums found for Release Radar - check cache_discovery_recent_albums()") + if recent_albums: # Group albums by artist for variety albums_by_artist = {} @@ -2196,7 +2294,9 @@ class WatchlistScanner: if source == 'spotify': album_data = self.spotify_client.get_album(album_id) else: - album_data = itunes_client.get_album(album_id) + album_data = itunes_api_call_with_retry( + itunes_client.get_album, album_id + ) if not album_data or 'tracks' not in album_data: continue @@ -2295,6 +2395,9 @@ class WatchlistScanner: logger.info(f"Curating Discovery Weekly for {source}...") discovery_tracks = self.database.get_discovery_pool_tracks(limit=2000, new_releases_only=False, source=source) + if not discovery_tracks: + logger.warning(f"[{source.upper()}] No discovery pool tracks found for Discovery Weekly - check populate_discovery_pool()") + discovery_weekly_tracks = [] if discovery_tracks: # Separate tracks by popularity tiers diff --git a/database/music_database.py b/database/music_database.py index 9a08b81b..2645e2ec 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -636,6 +636,90 @@ class MusicDatabase: cursor.execute("ALTER TABLE discovery_recent_albums ADD COLUMN source TEXT DEFAULT 'spotify'") logger.info("Added iTunes columns to discovery_recent_albums table for dual-source discovery") + # Migration: Fix NOT NULL constraint on album_spotify_id (required for iTunes-only albums) + # Check if album_spotify_id has NOT NULL constraint by checking table schema + cursor.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='discovery_recent_albums'") + table_schema = cursor.fetchone() + if table_schema and 'album_spotify_id TEXT NOT NULL' in (table_schema[0] or ''): + logger.info("Migrating discovery_recent_albums to allow NULL album_spotify_id for iTunes support...") + # SQLite doesn't support ALTER COLUMN, so recreate table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS discovery_recent_albums_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + album_spotify_id TEXT, + album_itunes_id TEXT, + artist_spotify_id TEXT, + artist_itunes_id TEXT, + source TEXT NOT NULL DEFAULT 'spotify', + album_name TEXT NOT NULL, + artist_name TEXT NOT NULL, + album_cover_url TEXT, + release_date TEXT NOT NULL, + album_type TEXT DEFAULT 'album', + cached_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(album_spotify_id, album_itunes_id, source) + ) + """) + cursor.execute(""" + INSERT OR IGNORE INTO discovery_recent_albums_new + SELECT * FROM discovery_recent_albums + """) + cursor.execute("DROP TABLE discovery_recent_albums") + cursor.execute("ALTER TABLE discovery_recent_albums_new RENAME TO discovery_recent_albums") + conn.commit() + logger.info("Successfully migrated discovery_recent_albums table for iTunes support") + + # Migration: Add UNIQUE constraint to similar_artists table + # Test if ON CONFLICT works by trying a dummy operation + needs_similar_migration = False + try: + cursor.execute(""" + INSERT INTO similar_artists + (source_artist_id, similar_artist_name, similarity_rank, occurrence_count, last_updated) + VALUES ('__migration_test__', '__migration_test__', 1, 1, CURRENT_TIMESTAMP) + ON CONFLICT(source_artist_id, similar_artist_name) + DO UPDATE SET occurrence_count = occurrence_count + """) + # Clean up test row + cursor.execute("DELETE FROM similar_artists WHERE source_artist_id = '__migration_test__'") + logger.info("similar_artists table has correct UNIQUE constraint") + except Exception as constraint_error: + logger.info(f"similar_artists needs migration (constraint test failed: {constraint_error})") + needs_similar_migration = True + + if needs_similar_migration: + logger.info("Migrating similar_artists to add UNIQUE constraint...") + # Get a fresh connection for the migration + with self._get_connection() as migration_conn: + migration_cursor = migration_conn.cursor() + # SQLite doesn't support adding constraints, so recreate table + migration_cursor.execute("DROP TABLE IF EXISTS similar_artists_new") + migration_cursor.execute(""" + CREATE TABLE similar_artists_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_artist_id TEXT NOT NULL, + similar_artist_spotify_id TEXT, + similar_artist_itunes_id TEXT, + similar_artist_name TEXT NOT NULL, + similarity_rank INTEGER DEFAULT 1, + occurrence_count INTEGER DEFAULT 1, + last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(source_artist_id, similar_artist_name) + ) + """) + migration_cursor.execute(""" + INSERT OR IGNORE INTO similar_artists_new + (source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, + similar_artist_name, similarity_rank, occurrence_count, last_updated) + SELECT source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, + similar_artist_name, similarity_rank, occurrence_count, last_updated + FROM similar_artists + """) + migration_cursor.execute("DROP TABLE similar_artists") + migration_cursor.execute("ALTER TABLE similar_artists_new RENAME TO similar_artists") + migration_conn.commit() + logger.info("Successfully migrated similar_artists table with UNIQUE constraint") + # ============== INDEXES (after migrations to ensure columns exist) ============== cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_source ON similar_artists (source_artist_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_spotify ON similar_artists (similar_artist_spotify_id)") diff --git a/tools/diagnose_itunes_discover.py b/tools/diagnose_itunes_discover.py new file mode 100644 index 00000000..f3b0a15b --- /dev/null +++ b/tools/diagnose_itunes_discover.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +""" +Diagnostic script to check iTunes data availability for the Discover page. + +Run this script to identify issues with iTunes data population: +- Similar artists missing iTunes IDs +- Discovery pool tracks by source +- Recent albums by source +- Curated playlists status + +Usage: + python tools/diagnose_itunes_discover.py +""" + +import sys +import os +import json + +# Add parent directory to path for imports +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from database.music_database import MusicDatabase + + +def diagnose_itunes_discover(): + """Run diagnostic checks for iTunes discover data.""" + + print("=" * 60) + print("iTunes Discover Page Diagnostic Report") + print("=" * 60) + + db = MusicDatabase() + + # 1. Check Similar Artists + print("\n[1] SIMILAR ARTISTS") + print("-" * 40) + + try: + with db._get_connection() as conn: + cursor = conn.cursor() + + # Total similar artists + cursor.execute("SELECT COUNT(*) as total FROM similar_artists") + total = cursor.fetchone()['total'] + + # With iTunes IDs + cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_itunes_id IS NOT NULL") + with_itunes = cursor.fetchone()['count'] + + # With Spotify IDs + cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_spotify_id IS NOT NULL") + with_spotify = cursor.fetchone()['count'] + + # With both + cursor.execute(""" + SELECT COUNT(*) as count FROM similar_artists + WHERE similar_artist_itunes_id IS NOT NULL + AND similar_artist_spotify_id IS NOT NULL + """) + with_both = cursor.fetchone()['count'] + + print(f" Total similar artists: {total}") + print(f" With iTunes ID: {with_itunes} ({100*with_itunes/total:.1f}%)" if total > 0 else " With iTunes ID: 0") + print(f" With Spotify ID: {with_spotify} ({100*with_spotify/total:.1f}%)" if total > 0 else " With Spotify ID: 0") + print(f" With BOTH IDs: {with_both} ({100*with_both/total:.1f}%)" if total > 0 else " With BOTH IDs: 0") + + if with_itunes == 0 and total > 0: + print(" [CRITICAL] No similar artists have iTunes IDs - Hero section will be empty!") + elif with_itunes < total * 0.5: + print(" [WARNING] Less than 50% of similar artists have iTunes IDs") + else: + print(" [OK] iTunes coverage is adequate") + + except Exception as e: + print(f" [ERROR] Could not check similar artists: {e}") + + # 2. Check Discovery Pool + print("\n[2] DISCOVERY POOL") + print("-" * 40) + + try: + with db._get_connection() as conn: + cursor = conn.cursor() + + # Total tracks + cursor.execute("SELECT COUNT(*) as total FROM discovery_pool") + total = cursor.fetchone()['total'] + + # By source + cursor.execute(""" + SELECT source, COUNT(*) as count + FROM discovery_pool + GROUP BY source + """) + source_counts = {row['source']: row['count'] for row in cursor.fetchall()} + + print(f" Total tracks: {total}") + print(f" Spotify tracks: {source_counts.get('spotify', 0)}") + print(f" iTunes tracks: {source_counts.get('itunes', 0)}") + + if source_counts.get('itunes', 0) == 0 and total > 0: + print(" [CRITICAL] No iTunes tracks in discovery pool - Fresh Tape/Archives will be empty!") + elif source_counts.get('itunes', 0) < total * 0.3: + print(" [WARNING] Low iTunes track count in discovery pool") + else: + print(" [OK] iTunes tracks present") + + except Exception as e: + print(f" [ERROR] Could not check discovery pool: {e}") + + # 3. Check Recent Albums + print("\n[3] RECENT ALBUMS CACHE") + print("-" * 40) + + try: + with db._get_connection() as conn: + cursor = conn.cursor() + + # Total albums + cursor.execute("SELECT COUNT(*) as total FROM discovery_recent_albums") + total = cursor.fetchone()['total'] + + # By source + cursor.execute(""" + SELECT source, COUNT(*) as count + FROM discovery_recent_albums + GROUP BY source + """) + source_counts = {row['source']: row['count'] for row in cursor.fetchall()} + + print(f" Total recent albums: {total}") + print(f" Spotify albums: {source_counts.get('spotify', 0)}") + print(f" iTunes albums: {source_counts.get('itunes', 0)}") + + if source_counts.get('itunes', 0) == 0 and total > 0: + print(" [CRITICAL] No iTunes albums cached - Recent Releases section will be empty!") + elif source_counts.get('itunes', 0) < 5: + print(" [WARNING] Very few iTunes albums cached") + else: + print(" [OK] iTunes albums cached") + + except Exception as e: + print(f" [ERROR] Could not check recent albums: {e}") + + # 4. Check Curated Playlists + print("\n[4] CURATED PLAYLISTS") + print("-" * 40) + + try: + with db._get_connection() as conn: + cursor = conn.cursor() + + playlists_to_check = [ + 'release_radar', + 'release_radar_spotify', + 'release_radar_itunes', + 'discovery_weekly', + 'discovery_weekly_spotify', + 'discovery_weekly_itunes' + ] + + for playlist_type in playlists_to_check: + cursor.execute(""" + SELECT track_ids_json FROM discovery_curated_playlists + WHERE playlist_type = ? + """, (playlist_type,)) + row = cursor.fetchone() + + if row: + track_ids = json.loads(row['track_ids_json']) + status = f"{len(track_ids)} tracks" + if len(track_ids) == 0: + status += " [EMPTY]" + else: + status = "[NOT FOUND]" + + print(f" {playlist_type}: {status}") + + # Check iTunes-specific playlists + cursor.execute(""" + SELECT track_ids_json FROM discovery_curated_playlists + WHERE playlist_type = 'release_radar_itunes' + """) + itunes_rr = cursor.fetchone() + + cursor.execute(""" + SELECT track_ids_json FROM discovery_curated_playlists + WHERE playlist_type = 'discovery_weekly_itunes' + """) + itunes_dw = cursor.fetchone() + + if not itunes_rr or len(json.loads(itunes_rr['track_ids_json'])) == 0: + print("\n [CRITICAL] release_radar_itunes is empty or missing!") + if not itunes_dw or len(json.loads(itunes_dw['track_ids_json'])) == 0: + print(" [CRITICAL] discovery_weekly_itunes is empty or missing!") + + except Exception as e: + print(f" [ERROR] Could not check curated playlists: {e}") + + # 5. Check Watchlist Artists + print("\n[5] WATCHLIST ARTISTS") + print("-" * 40) + + try: + with db._get_connection() as conn: + cursor = conn.cursor() + + # Total artists + cursor.execute("SELECT COUNT(*) as total FROM watchlist_artists") + total = cursor.fetchone()['total'] + + # With iTunes IDs + cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE itunes_artist_id IS NOT NULL") + with_itunes = cursor.fetchone()['count'] + + # With Spotify IDs + cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE spotify_artist_id IS NOT NULL") + with_spotify = cursor.fetchone()['count'] + + print(f" Total watchlist artists: {total}") + print(f" With iTunes ID: {with_itunes} ({100*with_itunes/total:.1f}%)" if total > 0 else " With iTunes ID: 0") + print(f" With Spotify ID: {with_spotify} ({100*with_spotify/total:.1f}%)" if total > 0 else " With Spotify ID: 0") + + if with_itunes == 0 and total > 0: + print(" [WARNING] No watchlist artists have iTunes IDs - source artist data limited") + + except Exception as e: + print(f" [ERROR] Could not check watchlist artists: {e}") + + # Summary + print("\n" + "=" * 60) + print("SUMMARY & RECOMMENDED ACTIONS") + print("=" * 60) + print(""" +If you see [CRITICAL] or [WARNING] messages above, follow these steps: + +QUICK FIX - Force Refresh Discover Data: +----------------------------------------- +Call the API endpoint to refresh discover data: + curl -X POST http://localhost:5000/api/discover/refresh + +This will: +- Cache recent albums from your watchlist artists +- Create curated playlists (Release Radar & Discovery Weekly) + +FULL FIX - Run Watchlist Scan: +------------------------------ +1. Go to the web UI Settings page +2. Click "Scan Watchlist" button +3. Wait for scan to complete + +This will: +- Fetch similar artists from MusicMap for each watchlist artist +- Populate the discovery pool with tracks +- Cache recent albums +- Create curated playlists + +ROOT CAUSE NOTES: +----------------- +- Similar artists = 0: MusicMap fetch may have failed. Watchlist scan needed. +- Recent albums = 0: cache_discovery_recent_albums() needs to run. +- Curated playlists missing: curate_discovery_playlists() needs to run. + +The discover page will now fall back to watchlist artists if similar +artists are not available, so basic functionality should still work. +""") + + +if __name__ == '__main__': + diagnose_itunes_discover() diff --git a/web_server.py b/web_server.py index 350abaad..0476b16b 100644 --- a/web_server.py +++ b/web_server.py @@ -14327,6 +14327,105 @@ def search_itunes_tracks(): return jsonify({"error": str(e)}), 500 +@app.route('/api/itunes/album/', methods=['GET']) +def get_itunes_album_tracks(album_id): + """Fetches full track details for a specific iTunes album.""" + try: + from core.itunes_client import iTunesClient + + itunes_client = iTunesClient() + album_data = itunes_client.get_album(album_id) + + if not album_data: + return jsonify({"error": "Album not found"}), 404 + + # Get tracks for this album + tracks_data = itunes_client.get_album_tracks(album_id) + tracks = tracks_data.get('items', []) if tracks_data else [] + + # Format response to match Spotify structure for frontend compatibility + album_dict = { + 'id': album_data.get('id', album_id), + 'name': album_data.get('name', 'Unknown Album'), + 'artists': album_data.get('artists', []), + 'release_date': album_data.get('release_date', ''), + 'total_tracks': album_data.get('total_tracks', len(tracks)), + 'album_type': album_data.get('album_type', 'album'), + 'images': album_data.get('images', []), + 'tracks': tracks, + 'source': 'itunes' + } + return jsonify(album_dict) + + except Exception as e: + logger.error(f"Error fetching iTunes album tracks: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/discover/album//', methods=['GET']) +def get_discover_album(source, album_id): + """ + Source-agnostic album endpoint for discover page. + Fetches album from the appropriate source (spotify or itunes). + """ + try: + if source == 'spotify': + if not spotify_client or not spotify_client.is_authenticated(): + return jsonify({"error": "Spotify not authenticated."}), 401 + + album_data = spotify_client.get_album(album_id) + if not album_data: + return jsonify({"error": "Album not found"}), 404 + + tracks = album_data.get('tracks', {}).get('items', []) + if not tracks: + tracks_data = spotify_client.get_album_tracks(album_id) + if tracks_data and 'items' in tracks_data: + tracks = tracks_data['items'] + + return jsonify({ + 'id': album_data['id'], + 'name': album_data['name'], + 'artists': album_data.get('artists', []), + 'release_date': album_data.get('release_date', ''), + 'total_tracks': album_data.get('total_tracks', 0), + 'album_type': album_data.get('album_type', 'album'), + 'images': album_data.get('images', []), + 'tracks': tracks, + 'source': 'spotify' + }) + + elif source == 'itunes': + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() + + album_data = itunes_client.get_album(album_id) + if not album_data: + return jsonify({"error": "Album not found"}), 404 + + tracks_data = itunes_client.get_album_tracks(album_id) + tracks = tracks_data.get('items', []) if tracks_data else [] + + return jsonify({ + 'id': album_data.get('id', album_id), + 'name': album_data.get('name', 'Unknown Album'), + 'artists': album_data.get('artists', []), + 'release_date': album_data.get('release_date', ''), + 'total_tracks': album_data.get('total_tracks', len(tracks)), + 'album_type': album_data.get('album_type', 'album'), + 'images': album_data.get('images', []), + 'tracks': tracks, + 'source': 'itunes' + }) + + else: + return jsonify({"error": f"Unknown source: {source}"}), 400 + + except Exception as e: + logger.error(f"Error fetching discover album: {e}") + return jsonify({"error": str(e)}), 500 + + # =================================================================== # TIDAL PLAYLIST API ENDPOINTS # =================================================================== @@ -17618,7 +17717,24 @@ def start_watchlist_scan(): })()) print(f"βœ… Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist") - + + # Fetch similar artists for discovery feature + # This is critical for the discover page to work + try: + watchlist_scan_state['current_phase'] = 'fetching_similar_artists' + source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id) + + if database.has_fresh_similar_artists(source_artist_id, days_threshold=30): + print(f" Similar artists for {artist.artist_name} are cached and fresh") + # Still backfill missing iTunes IDs + scanner._backfill_similar_artists_itunes_ids(source_artist_id) + else: + print(f" Fetching similar artists for {artist.artist_name}...") + scanner.update_similar_artists(artist) + print(f" Similar artists updated for {artist.artist_name}") + except Exception as similar_error: + print(f" ⚠️ Failed to update similar artists for {artist.artist_name}: {similar_error}") + # Delay between artists if i < len(watchlist_artists) - 1: watchlist_scan_state['current_phase'] = 'rate_limiting' @@ -18469,11 +18585,63 @@ def get_discover_hero(): active_source = _get_active_discovery_source() print(f"🎡 Discover hero using source: {active_source}") + # Import iTunes client for fallback + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() + # Get top similar artists (by occurrence count) - get 20 for variety similar_artists = database.get_top_similar_artists(limit=20) + # FALLBACK: If no similar artists exist, use watchlist artists for Hero section if not similar_artists: - return jsonify({"success": True, "artists": [], "source": active_source}) + print("[Discover Hero] No similar artists found, falling back to watchlist artists") + watchlist_artists = database.get_watchlist_artists() + + if not watchlist_artists: + return jsonify({"success": True, "artists": [], "source": active_source}) + + # Convert watchlist artists to hero format + import random + shuffled_watchlist = list(watchlist_artists) + random.shuffle(shuffled_watchlist) + + hero_artists = [] + for artist in shuffled_watchlist[:10]: + artist_id = artist.itunes_artist_id if active_source == 'itunes' else artist.spotify_artist_id + if not artist_id: + continue + + artist_data = { + "spotify_artist_id": artist.spotify_artist_id, + "itunes_artist_id": artist.itunes_artist_id, + "artist_id": artist_id, + "artist_name": artist.artist_name, + "occurrence_count": 1, + "similarity_rank": 1, + "source": active_source, + "is_watchlist": True + } + + # Try to get artist image + try: + if active_source == 'itunes' and artist.itunes_artist_id: + itunes_artist = itunes_client.get_artist(artist.itunes_artist_id) + if itunes_artist: + artist_data['image_url'] = itunes_artist.get('images', [{}])[0].get('url') if itunes_artist.get('images') else None + artist_data['genres'] = itunes_artist.get('genres', []) + elif active_source == 'spotify' and artist.spotify_artist_id: + if spotify_client and spotify_client.is_authenticated(): + sp_artist = spotify_client.get_artist(artist.spotify_artist_id) + if sp_artist and sp_artist.get('images'): + artist_data['image_url'] = sp_artist['images'][0]['url'] if sp_artist['images'] else None + artist_data['genres'] = sp_artist.get('genres', []) + except Exception as img_err: + print(f"Could not fetch watchlist artist image: {img_err}") + + hero_artists.append(artist_data) + + print(f"[Discover Hero] Returning {len(hero_artists)} watchlist artists as fallback") + return jsonify({"success": True, "artists": hero_artists, "source": active_source, "fallback": "watchlist"}) # Filter to artists that have the appropriate ID for the active source valid_artists = [] @@ -18486,16 +18654,41 @@ def get_discover_hero(): elif artist.similar_artist_spotify_id and artist.similar_artist_itunes_id: valid_artists.append(artist) + # FALLBACK: If no valid artists for iTunes, try to resolve iTunes IDs on-the-fly + if active_source == 'itunes' and not valid_artists: + print(f"[iTunes Fallback] No artists with iTunes IDs found, attempting on-the-fly resolution for {len(similar_artists)} artists") + resolved_count = 0 + for artist in similar_artists: + if artist.similar_artist_itunes_id: + valid_artists.append(artist) + continue + # Try to resolve iTunes ID by name + try: + itunes_results = itunes_client.search_artists(artist.similar_artist_name, limit=1) + if itunes_results and len(itunes_results) > 0: + itunes_id = itunes_results[0].id + # Cache the resolved ID for future use + database.update_similar_artist_itunes_id(artist.id, itunes_id) + # Create a modified artist object with the resolved ID + artist.similar_artist_itunes_id = itunes_id + valid_artists.append(artist) + resolved_count += 1 + print(f" [Resolved] {artist.similar_artist_name} -> iTunes ID: {itunes_id}") + except Exception as resolve_err: + print(f" [Failed] Could not resolve iTunes ID for {artist.similar_artist_name}: {resolve_err}") + # Stop after 10 successful resolutions to avoid rate limiting + if len(valid_artists) >= 10: + break + print(f"[iTunes Fallback] Resolved {resolved_count} artists with iTunes IDs") + + print(f"[Discover Hero] Found {len(valid_artists)} valid artists for source: {active_source}") + # Shuffle for variety and take top 10 import random shuffled = list(valid_artists) random.shuffle(shuffled) similar_artists = shuffled[:10] - # Import iTunes client for fallback - from core.itunes_client import iTunesClient - itunes_client = iTunesClient() - # Convert to JSON format with data enrichment from appropriate source hero_artists = [] for artist in similar_artists: @@ -18685,6 +18878,121 @@ def get_discover_weekly(): print(f"Error getting discovery weekly: {e}") return jsonify({"success": False, "error": str(e)}), 500 + +@app.route('/api/discover/refresh', methods=['POST']) +def refresh_discover_data(): + """ + Force refresh discover page data (recent albums cache and curated playlists). + Useful for initial setup or when data appears stale. + """ + try: + from core.watchlist_scanner import WatchlistScanner + + database = get_database() + scanner = WatchlistScanner(spotify_client, database) + + print("[Discover Refresh] Starting forced refresh of discover data...") + + # Cache recent albums from watchlist and similar artists + print("[Discover Refresh] Caching recent albums...") + scanner.cache_discovery_recent_albums() + + # Curate playlists + print("[Discover Refresh] Curating discovery playlists...") + scanner.curate_discovery_playlists() + + # Get counts for response + active_source = _get_active_discovery_source() + recent_albums = database.get_discovery_recent_albums(limit=100, source=active_source) + release_radar = database.get_curated_playlist(f'release_radar_{active_source}') or [] + discovery_weekly = database.get_curated_playlist(f'discovery_weekly_{active_source}') or [] + + print(f"[Discover Refresh] Complete! Recent albums: {len(recent_albums)}, Release Radar: {len(release_radar)} tracks, Discovery Weekly: {len(discovery_weekly)} tracks") + + return jsonify({ + "success": True, + "message": "Discover data refreshed", + "source": active_source, + "recent_albums_count": len(recent_albums), + "release_radar_tracks": len(release_radar), + "discovery_weekly_tracks": len(discovery_weekly) + }) + + except Exception as e: + print(f"Error refreshing discover data: {e}") + import traceback + traceback.print_exc() + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/discover/diagnose', methods=['GET']) +def diagnose_discover_data(): + """ + Diagnostic endpoint to check the state of discover data. + Returns counts of similar artists, discovery pool, recent albums, etc. + """ + try: + database = get_database() + active_source = _get_active_discovery_source() + + with database._get_connection() as conn: + cursor = conn.cursor() + + # Similar artists stats + cursor.execute("SELECT COUNT(*) as total FROM similar_artists") + total_similar = cursor.fetchone()['total'] + + cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_itunes_id IS NOT NULL") + similar_with_itunes = cursor.fetchone()['count'] + + cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_spotify_id IS NOT NULL") + similar_with_spotify = cursor.fetchone()['count'] + + # Discovery pool stats + cursor.execute("SELECT source, COUNT(*) as count FROM discovery_pool GROUP BY source") + pool_by_source = {row['source']: row['count'] for row in cursor.fetchall()} + + # Recent albums stats + cursor.execute("SELECT source, COUNT(*) as count FROM discovery_recent_albums GROUP BY source") + albums_by_source = {row['source']: row['count'] for row in cursor.fetchall()} + + # Curated playlists + cursor.execute("SELECT playlist_type, track_ids_json FROM discovery_curated_playlists") + playlists = {} + for row in cursor.fetchall(): + import json + track_ids = json.loads(row['track_ids_json']) if row['track_ids_json'] else [] + playlists[row['playlist_type']] = len(track_ids) + + # Watchlist artists + cursor.execute("SELECT COUNT(*) as total FROM watchlist_artists") + total_watchlist = cursor.fetchone()['total'] + + cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE itunes_artist_id IS NOT NULL") + watchlist_with_itunes = cursor.fetchone()['count'] + + return jsonify({ + "success": True, + "active_source": active_source, + "similar_artists": { + "total": total_similar, + "with_itunes_id": similar_with_itunes, + "with_spotify_id": similar_with_spotify + }, + "discovery_pool": pool_by_source, + "recent_albums": albums_by_source, + "curated_playlists": playlists, + "watchlist_artists": { + "total": total_watchlist, + "with_itunes_id": watchlist_with_itunes + } + }) + + except Exception as e: + print(f"Error diagnosing discover data: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + # ======================================== # SEASONAL DISCOVERY ENDPOINTS # ======================================== diff --git a/webui/static/script.js b/webui/static/script.js index 98cd6e44..8e6e1a2b 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -33121,8 +33121,16 @@ async function openDownloadModalForRecentAlbum(albumIndex) { showLoadingOverlay(`Loading tracks for ${album.album_name}...`); try { - // Fetch album tracks from Spotify API via backend - const response = await fetch(`/api/spotify/album/${album.album_spotify_id}`); + // Determine source and album ID - use source-agnostic endpoint + const source = album.source || (album.album_spotify_id ? 'spotify' : 'itunes'); + const albumId = source === 'spotify' ? album.album_spotify_id : album.album_itunes_id; + + if (!albumId) { + throw new Error(`No ${source} album ID available`); + } + + // Fetch album tracks from appropriate source via backend + const response = await fetch(`/api/discover/album/${source}/${albumId}`); if (!response.ok) { throw new Error('Failed to fetch album tracks'); } @@ -33157,13 +33165,14 @@ async function openDownloadModalForRecentAlbum(albumIndex) { }; }); - // Create virtual playlist ID - const virtualPlaylistId = `discover_album_${album.album_spotify_id}`; + // Create virtual playlist ID using the appropriate album ID + const virtualPlaylistId = `discover_album_${albumId}`; // CRITICAL FIX: Pass proper artist/album context for modal display const artistContext = { - id: album.artist_spotify_id, - name: album.artist_name + id: source === 'spotify' ? album.artist_spotify_id : album.artist_itunes_id, + name: album.artist_name, + source: source }; const albumContext = { From 4f1dc2c15f4ea3c96ec5c66140642969f40f979b Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sat, 24 Jan 2026 23:46:01 -0800 Subject: [PATCH 14/20] force refetch similar artists when Spotify IDs missing When Spotify is enabled after populating similar artists with only iTunes IDs, the freshness check now detects missing Spotify IDs and triggers a refetch. This fixes the Discover page not showing data when switching from iTunes-only mode. --- core/watchlist_scanner.py | 4 +++- database/music_database.py | 22 ++++++++++++++++++++-- web_server.py | 4 +++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 367ebddf..7b5f96e2 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -685,7 +685,9 @@ class WatchlistScanner: source_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or str(watchlist_artist.id) try: # Check if we have fresh similar artists cached (< 30 days old) - if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30): + # If Spotify is authenticated, also require Spotify IDs to be present + spotify_authenticated = self.spotify_client and self.spotify_client.is_spotify_authenticated() + if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated): logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping MusicMap fetch") # Even if cached, backfill missing iTunes IDs (seamless dual-source support) self._backfill_similar_artists_itunes_ids(source_artist_id) diff --git a/database/music_database.py b/database/music_database.py index 2645e2ec..e06b232b 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -3212,15 +3212,16 @@ class MusicDatabase: logger.error(f"Error updating similar artist iTunes ID: {e}") return False - def has_fresh_similar_artists(self, source_artist_id: str, days_threshold: int = 30, require_itunes: bool = True) -> bool: + def has_fresh_similar_artists(self, source_artist_id: str, days_threshold: int = 30, require_itunes: bool = True, require_spotify: bool = False) -> bool: """ - Check if we have cached similar artists that are still fresh (< days_threshold old). + Check if we have cached similar artists that are still fresh ( 0: + # If less than 50% have Spotify IDs, consider stale and refetch + spotify_ratio = id_row['has_spotify'] / id_row['total'] + if spotify_ratio < 0.5: + logger.debug(f"Similar artists for {source_artist_id} missing Spotify IDs ({id_row['has_spotify']}/{id_row['total']}), will refetch") + return False + return True except Exception as e: diff --git a/web_server.py b/web_server.py index 0476b16b..6448b616 100644 --- a/web_server.py +++ b/web_server.py @@ -17724,7 +17724,9 @@ def start_watchlist_scan(): watchlist_scan_state['current_phase'] = 'fetching_similar_artists' source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id) - if database.has_fresh_similar_artists(source_artist_id, days_threshold=30): + # If Spotify is authenticated, also require Spotify IDs to be present + spotify_authenticated = spotify_client and spotify_client.is_spotify_authenticated() + if database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated): print(f" Similar artists for {artist.artist_name} are cached and fresh") # Still backfill missing iTunes IDs scanner._backfill_similar_artists_itunes_ids(source_artist_id) From 4cbb3c952b59f997a648ad8e2b04035f56d13fa4 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sat, 24 Jan 2026 23:58:08 -0800 Subject: [PATCH 15/20] Fix 'view discog' button on discover hero slider when itunes is primary source. --- webui/static/script.js | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/webui/static/script.js b/webui/static/script.js index 8e6e1a2b..039baa60 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -18616,8 +18616,8 @@ function updateYouTubeDiscoveryModal(urlHash, status) { if (actionsCell) { const state = listenbrainzPlaylistStates[urlHash] || youtubePlaylistStates[urlHash]; const platform = state?.is_listenbrainz_playlist ? 'listenbrainz' : - (state?.is_tidal_playlist ? 'tidal' : - (state?.is_beatport_playlist ? 'beatport' : 'youtube')); + (state?.is_tidal_playlist ? 'tidal' : + (state?.is_beatport_playlist ? 'beatport' : 'youtube')); actionsCell.innerHTML = generateDiscoveryActionButton(result, urlHash, platform); } }); @@ -30065,20 +30065,28 @@ function displayDiscoverHeroArtist(artist) { } // Store artist ID for both buttons and update watchlist state + // Use artist_id which is set by the backend to the appropriate ID for the active source const addBtn = document.getElementById('discover-hero-add'); const discographyBtn = document.getElementById('discover-hero-discography'); + const artistId = artist.artist_id || artist.spotify_artist_id || artist.itunes_artist_id; - if (addBtn && artist.spotify_artist_id) { - addBtn.setAttribute('data-artist-id', artist.spotify_artist_id); + if (addBtn && artistId) { + addBtn.setAttribute('data-artist-id', artistId); addBtn.setAttribute('data-artist-name', artist.artist_name); + // Also store both IDs for cross-source operations + if (artist.spotify_artist_id) addBtn.setAttribute('data-spotify-id', artist.spotify_artist_id); + if (artist.itunes_artist_id) addBtn.setAttribute('data-itunes-id', artist.itunes_artist_id); // Check if this artist is already in watchlist and update button appearance - checkAndUpdateDiscoverHeroWatchlistButton(artist.spotify_artist_id); + checkAndUpdateDiscoverHeroWatchlistButton(artistId); } - if (discographyBtn && artist.spotify_artist_id) { - discographyBtn.setAttribute('data-artist-id', artist.spotify_artist_id); + if (discographyBtn && artistId) { + discographyBtn.setAttribute('data-artist-id', artistId); discographyBtn.setAttribute('data-artist-name', artist.artist_name); + // Also store both IDs for cross-source operations + if (artist.spotify_artist_id) discographyBtn.setAttribute('data-spotify-id', artist.spotify_artist_id); + if (artist.itunes_artist_id) discographyBtn.setAttribute('data-itunes-id', artist.itunes_artist_id); } // Update slideshow indicators From 59848acaf3866fce38ee495a1331dea8603a542b Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sun, 25 Jan 2026 00:29:33 -0800 Subject: [PATCH 16/20] feat: dynamic source labels and enhanced connection testing - Display "Apple Music" instead of "Spotify" in UI when iTunes is active source - Enhanced connection test messages to indicate Spotify config/auth status - Fixed similar artists requiring Spotify re-scan when Spotify becomes available - Fixed hero slider buttons failing for iTunes-only artists - Updated activity feed items to show correct source name dynamically --- web_server.py | 28 +++++++++++++++++++++++----- webui/index.html | 4 ++-- webui/static/script.js | 24 ++++++++++++++++++++++-- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/web_server.py b/web_server.py index 6448b616..2ec13b97 100644 --- a/web_server.py +++ b/web_server.py @@ -1506,10 +1506,23 @@ def run_service_test(service, test_config): # 3. Run the test with the temporary config if service == "spotify": temp_client = SpotifyClient() + + # Check if Spotify credentials are configured + spotify_config = config_manager.get('spotify', {}) + spotify_configured = bool(spotify_config.get('client_id') and spotify_config.get('client_secret')) + if temp_client.is_authenticated(): - return True, "Spotify connection successful!" + # Determine which source is active + if temp_client.is_spotify_authenticated(): + return True, "Spotify connection successful!" + else: + # Using iTunes fallback + if spotify_configured: + return True, "Apple Music connection successful! (Spotify configured but not authenticated)" + else: + return True, "Apple Music connection successful! (Spotify not configured)" else: - return False, "Spotify authentication failed. Check credentials and complete OAuth flow in browser if prompted." + return False, "Music service authentication failed. Check credentials and complete OAuth flow in browser if prompted." elif service == "tidal": temp_client = TidalClient() if temp_client.is_authenticated(): @@ -1926,9 +1939,14 @@ def get_status(): # Actually validate authentication (makes API call, but cached for 2 min) spotify_status = spotify_client.is_authenticated() spotify_response_time = (time.time() - spotify_start) * 1000 + + # Determine active music source (spotify or itunes) + music_source = 'spotify' if spotify_client.is_spotify_authenticated() else 'itunes' + _status_cache['spotify'] = { 'connected': spotify_status, - 'response_time': round(spotify_response_time, 1) + 'response_time': round(spotify_response_time, 1), + 'source': music_source } _status_cache_timestamps['spotify'] = current_time # else: use cached value @@ -2365,7 +2383,7 @@ def test_connection_endpoint(): # Add activity for connection test if success: - add_activity_item("βœ…", "Connection Test", f"{service.title()} connection successful", "Now") + add_activity_item("βœ…", "Connection Test", message, "Now") else: add_activity_item("❌", "Connection Test", f"{service.title()} connection failed", "Now") @@ -2411,7 +2429,7 @@ def test_dashboard_connection_endpoint(): # Add activity for dashboard connection test (different from settings test) if success: - add_activity_item("πŸŽ›οΈ", "Dashboard Test", f"{service.title()} service verified", "Now") + add_activity_item("πŸŽ›οΈ", "Dashboard Test", message, "Now") else: add_activity_item("⚠️", "Dashboard Test", f"{service.title()} service check failed", "Now") diff --git a/webui/index.html b/webui/index.html index 0f2a0ea2..34db2301 100644 --- a/webui/index.html +++ b/webui/index.html @@ -139,7 +139,7 @@

Service Status

- Spotify + Spotify
@@ -175,7 +175,7 @@
- Spotify + Spotify ●
diff --git a/webui/static/script.js b/webui/static/script.js index 039baa60..137dcb1a 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -2164,7 +2164,8 @@ async function testConnection(service) { const result = await response.json(); if (result.success) { - showToast(`${service} connection successful`, 'success'); + // Use backend's message which contains dynamic source name (Spotify or Apple Music) + showToast(result.message || `${service} connection successful`, 'success'); // Load music libraries after successful connection if (service === 'plex') { @@ -2197,7 +2198,8 @@ async function testDashboardConnection(service) { const result = await response.json(); if (result.success) { - showToast(`${service} service verified`, 'success'); + // Use backend's message which contains dynamic source name (Spotify or Apple Music) + showToast(result.message || `${service} service verified`, 'success'); } else { showToast(`${service} service check failed: ${result.error}`, 'error'); } @@ -23214,6 +23216,15 @@ function updateServiceStatus(service, statusData) { statusText.className = 'service-card-status-text disconnected'; } } + + // Update music source title (Spotify or Apple Music) based on active source + if (service === 'spotify' && statusData.source) { + const musicSourceTitleElement = document.getElementById('music-source-title'); + if (musicSourceTitleElement) { + const sourceName = statusData.source === 'itunes' ? 'Apple Music' : 'Spotify'; + musicSourceTitleElement.textContent = sourceName; + } + } } function updateSidebarServiceStatus(service, statusData) { @@ -23238,6 +23249,15 @@ function updateSidebarServiceStatus(service, statusData) { mediaServerNameElement.textContent = serverName; } } + + // Update music source name (Spotify or Apple Music) based on active source + if (service === 'spotify' && statusData.source) { + const musicSourceNameElement = document.getElementById('music-source-name'); + if (musicSourceNameElement) { + const sourceName = statusData.source === 'itunes' ? 'Apple Music' : 'Spotify'; + musicSourceNameElement.textContent = sourceName; + } + } } } From 47c45ddea70750ac25049b340b7baf8c74326fdf Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sun, 25 Jan 2026 00:35:16 -0800 Subject: [PATCH 17/20] feat: dynamic music source labels in discovery modals - Added global currentMusicSourceName variable to track active source - Updated discovery modal to show "Apple Music" when iTunes is active - Replaced hardcoded "Spotify" in modal titles, headers, and descriptions - Discovery modals now automatically reflect the correct music source (Spotify/Apple Music) --- webui/static/script.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/webui/static/script.js b/webui/static/script.js index 137dcb1a..c6cf4a64 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -12,6 +12,7 @@ let currentStream = { progress: 0, track: null }; +let currentMusicSourceName = 'Spotify'; // 'Spotify' or 'Apple Music' - updated from status endpoint // Streaming state management (enhanced functionality) let streamStatusPoller = null; @@ -18137,7 +18138,7 @@ function openYouTubeDiscoveryModal(urlHash) {
-
πŸ” Spotify Discovery Progress
+
πŸ” ${currentMusicSourceName} Discovery Progress
@@ -18151,8 +18152,8 @@ function openYouTubeDiscoveryModal(urlHash) { ${sourceLabel} Track ${sourceLabel} Artist Status - Spotify Track - Spotify Artist + ${currentMusicSourceName} Track + ${currentMusicSourceName} Artist Album Actions @@ -18300,7 +18301,7 @@ function getModalActionButtons(urlHash, phase, state = null) { } } else { // Discovering phase - show progress - return `
πŸ” Discovering Spotify matches...
`; + return `
πŸ” Discovering ${currentMusicSourceName} matches...
`; } case 'discovered': @@ -18446,13 +18447,13 @@ function getModalDescription(phase, isTidal = false, isBeatport = false, isListe const source = isListenBrainz ? 'ListenBrainz' : (isBeatport ? 'Beatport' : (isTidal ? 'Tidal' : 'YouTube')); switch (phase) { case 'fresh': - return `Ready to discover clean Spotify metadata for ${source} tracks...`; + return `Ready to discover clean ${currentMusicSourceName} metadata for ${source} tracks...`; case 'discovering': - return `Discovering clean Spotify metadata for ${source} tracks...`; + return `Discovering clean ${currentMusicSourceName} metadata for ${source} tracks...`; case 'discovered': return 'Discovery complete! View the results below.'; default: - return `Discovering clean Spotify metadata for ${source} tracks...`; + return `Discovering clean ${currentMusicSourceName} metadata for ${source} tracks...`; } } @@ -23223,6 +23224,8 @@ function updateServiceStatus(service, statusData) { if (musicSourceTitleElement) { const sourceName = statusData.source === 'itunes' ? 'Apple Music' : 'Spotify'; musicSourceTitleElement.textContent = sourceName; + // Update global variable for use in discovery modals + currentMusicSourceName = sourceName; } } } From 905e98016f1188e1795680fa0f3f90c19874407b Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sun, 25 Jan 2026 09:24:28 -0800 Subject: [PATCH 18/20] Fix watchlist artist config and add image at insert when Itunes source. --- database/music_database.py | 64 ++++++++++++++++++++++++++++++++++++++ web_server.py | 61 +++++++++++++++++++++++++++++------- webui/static/script.js | 4 +-- 3 files changed, 116 insertions(+), 13 deletions(-) diff --git a/database/music_database.py b/database/music_database.py index e06b232b..bf28fc7a 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -291,6 +291,9 @@ class MusicDatabase: # Add iTunes artist ID column to watchlist_artists (migration) self._add_watchlist_itunes_id_column(cursor) + # Make spotify_artist_id nullable for iTunes-only artists (migration) + self._fix_watchlist_spotify_id_nullable(cursor) + conn.commit() logger.info("Database initialized successfully") @@ -821,6 +824,67 @@ class MusicDatabase: logger.error(f"Error adding itunes_artist_id column to watchlist_artists: {e}") # Don't raise - this is a migration, database can still function + def _fix_watchlist_spotify_id_nullable(self, cursor): + """ + Make spotify_artist_id nullable in watchlist_artists table. + This allows adding iTunes-only artists without Spotify IDs. + + Since SQLite doesn't support modifying column constraints directly, + we need to recreate the table if the constraint needs to be changed. + """ + try: + # Check if spotify_artist_id is currently NOT NULL + cursor.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='watchlist_artists'") + result = cursor.fetchone() + + if result and 'spotify_artist_id TEXT UNIQUE NOT NULL' in result[0]: + logger.info("Migrating watchlist_artists table to make spotify_artist_id nullable...") + + # Create new table with nullable spotify_artist_id + cursor.execute(""" + CREATE TABLE watchlist_artists_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + spotify_artist_id TEXT UNIQUE, + artist_name TEXT NOT NULL, + date_added TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + last_scan_timestamp TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + image_url TEXT, + include_albums INTEGER DEFAULT 1, + include_eps INTEGER DEFAULT 1, + include_singles INTEGER DEFAULT 1, + include_live INTEGER DEFAULT 0, + include_remixes INTEGER DEFAULT 0, + include_acoustic INTEGER DEFAULT 0, + include_compilations INTEGER DEFAULT 0, + itunes_artist_id TEXT + ) + """) + + # Copy data from old table + cursor.execute(""" + INSERT INTO watchlist_artists_new + SELECT * FROM watchlist_artists + """) + + # Drop old table + cursor.execute("DROP TABLE watchlist_artists") + + # Rename new table + cursor.execute("ALTER TABLE watchlist_artists_new RENAME TO watchlist_artists") + + # Recreate indexes + cursor.execute("CREATE INDEX IF NOT EXISTS idx_watchlist_spotify_id ON watchlist_artists (spotify_artist_id)") + + logger.info("Successfully migrated watchlist_artists table - spotify_artist_id is now nullable") + else: + logger.debug("watchlist_artists table already has nullable spotify_artist_id or custom schema") + + except Exception as e: + logger.error(f"Error making spotify_artist_id nullable in watchlist_artists: {e}") + # Don't raise - this is a migration, database can still function + def close(self): """Close database connection (no-op since we create connections per operation)""" # Each operation creates and closes its own connection, so nothing to do here diff --git a/web_server.py b/web_server.py index 2ec13b97..d74e6a0e 100644 --- a/web_server.py +++ b/web_server.py @@ -17423,7 +17423,8 @@ def get_watchlist_artists(): "last_scan_timestamp": artist.last_scan_timestamp.isoformat() if artist.last_scan_timestamp else None, "created_at": artist.created_at.isoformat() if artist.created_at else None, "updated_at": artist.updated_at.isoformat() if artist.updated_at else None, - "image_url": artist.image_url # Cached during watchlist scans + "image_url": artist.image_url, # Cached during watchlist scans + "itunes_artist_id": artist.itunes_artist_id # For iTunes-only artists }) return jsonify({"success": True, "artists": artists_data}) @@ -17446,9 +17447,42 @@ def add_to_watchlist(): success = database.add_artist_to_watchlist(artist_id, artist_name) if success: - # Fetch and cache artist image immediately from Spotify + # Detect ID type: iTunes IDs are purely numeric, Spotify IDs are alphanumeric + is_itunes_id = artist_id.isdigit() + + # Fetch and cache artist image immediately try: - if spotify_client and spotify_client.is_authenticated(): + if is_itunes_id: + # For iTunes artists, fetch image from iTunes API + # We look up 'album' entity because 'artist' entity doesn't always have artwork + # We fallback to the first album's artwork as the artist image + try: + itunes_url = f"https://itunes.apple.com/lookup?id={artist_id}&entity=album&limit=5" + print(f"πŸ” Fetching iTunes artist image: {itunes_url}") + resp = requests.get(itunes_url, timeout=5) + + image_url = None + if resp.status_code == 200: + data = resp.json() + results = data.get('results', []) + + # Iterate results to find one with artwork + for res in results: + if 'artworkUrl100' in res: + # Get highest res by replacing dimensions + # iTunes artwork URLs usually end in .../100x100bb.jpg + image_url = res['artworkUrl100'].replace('100x100', '600x600') + break + + if image_url: + database.update_watchlist_artist_image(artist_id, image_url) + print(f"βœ… Cached iTunes artist image for {artist_name}") + else: + print(f"⚠️ No artwork found for iTunes artist {artist_name}") + except Exception as itunes_error: + print(f"⚠️ Error fetching iTunes artwork: {itunes_error}") + elif spotify_client and spotify_client.is_authenticated(): + # For Spotify artists, fetch from Spotify API artist_data = spotify_client.get_artist(artist_id) if artist_data and 'images' in artist_data and artist_data['images']: # Get medium-sized image (usually the second one, or first if only one) @@ -17974,21 +18008,26 @@ def watchlist_artist_config(artist_id): cursor.execute(""" SELECT include_albums, include_eps, include_singles, include_live, include_remixes, include_acoustic, include_compilations, - artist_name, image_url + artist_name, image_url, spotify_artist_id, itunes_artist_id FROM watchlist_artists - WHERE spotify_artist_id = ? - """, (artist_id,)) + WHERE spotify_artist_id = ? OR itunes_artist_id = ? + """, (artist_id, artist_id)) result = cursor.fetchone() conn.close() if not result: return jsonify({"success": False, "error": "Artist not found in watchlist"}), 404 + + # Determine if this is an iTunes or Spotify artist + is_itunes_artist = artist_id.isdigit() + spotify_id = result[9] # spotify_artist_id from query + itunes_id = result[10] # itunes_artist_id from query - # Get artist info from Spotify + # Get artist info from Spotify (only for Spotify artists) artist_info = None - if spotify_client and spotify_client.is_authenticated(): + if not is_itunes_artist and spotify_client and spotify_client.is_authenticated() and spotify_id: try: - artist_data = spotify_client.sp.artist(artist_id) + artist_data = spotify_client.sp.artist(spotify_id) if artist_data: artist_info = { 'id': artist_data['id'], @@ -18053,10 +18092,10 @@ def watchlist_artist_config(artist_id): SET include_albums = ?, include_eps = ?, include_singles = ?, include_live = ?, include_remixes = ?, include_acoustic = ?, include_compilations = ?, updated_at = CURRENT_TIMESTAMP - WHERE spotify_artist_id = ? + WHERE spotify_artist_id = ? OR itunes_artist_id = ? """, (int(include_albums), int(include_eps), int(include_singles), int(include_live), int(include_remixes), int(include_acoustic), int(include_compilations), - artist_id)) + artist_id, artist_id)) conn.commit() if cursor.rowcount == 0: diff --git a/webui/static/script.js b/webui/static/script.js index c6cf4a64..9db125a2 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -23669,7 +23669,7 @@ async function showWatchlistModal() { ${artistsData.artists.map(artist => `
${artist.image_url ? ` +
@@ -3305,7 +3305,8 @@

Content Filters

-

Check to INCLUDE, leave unchecked to EXCLUDE (default: all excluded)

+

Check to INCLUDE, leave unchecked to EXCLUDE (default: all + excluded)

@@ -3325,7 +3327,8 @@
🎧
Include Remixes - Check to include remix versions and edits + Check to include remix versions and + edits
@@ -3336,7 +3339,8 @@
🎸
Include Acoustic Versions - Check to include acoustic and stripped versions + Check to include acoustic and stripped + versions
@@ -3347,7 +3351,8 @@
πŸ“€
Include Compilations - Check to include greatest hits and collections + Check to include greatest hits and + collections
From f4365fa8361d6e399dce77bca65aa450dcfcaf37 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sun, 25 Jan 2026 10:51:02 -0800 Subject: [PATCH 20/20] Fix artist image not appearing for artist bubble, artist bubble modal or when an artist is selected in search. --- webui/static/script.js | 47 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/webui/static/script.js b/webui/static/script.js index 9db125a2..b3e4a733 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -21052,8 +21052,24 @@ function updateArtistDetailHeader(artist) { const nameElement = document.getElementById('search-artist-detail-name'); const genresElement = document.getElementById('search-artist-detail-genres'); - if (imageElement && artist.image_url) { - imageElement.style.backgroundImage = `url('${artist.image_url}')`; + if (imageElement) { + if (artist.image_url) { + imageElement.style.backgroundImage = `url('${artist.image_url}')`; + } else { + // Lazy load image if missing (common for iTunes artists) + console.log(`πŸ–ΌοΈ Lazy loading detail image for ${artist.name} (${artist.id})`); + fetch(`/api/artist/${artist.id}/image`) + .then(response => response.json()) + .then(data => { + if (data.success && data.image_url) { + console.log(`βœ… Loaded detail image for ${artist.name}`); + imageElement.style.backgroundImage = `url('${data.image_url}')`; + // Update the artist object in memory too + artist.image_url = data.image_url; + } + }) + .catch(err => console.error('❌ Failed to load detail image:', err)); + } } if (nameElement) { @@ -22174,8 +22190,35 @@ async function openSearchDownloadModal(artistName) { document.body.appendChild(modal); modal.style.display = 'flex'; + // Start monitoring for status changes // Start monitoring for status changes monitorSearchDownloadModal(artistName); + + // Lazy load artist image if missing (common for iTunes) + if (!artistBubbleData.artist.image_url) { + console.log(`πŸ–ΌοΈ Lazy loading modal image for ${artistBubbleData.artist.name} (${artistBubbleData.artist.id})`); + fetch(`/api/artist/${artistBubbleData.artist.id}/image`) + .then(response => response.json()) + .then(data => { + if (data.success && data.image_url) { + // Update header background + const headerBg = modal.querySelector('.artist-download-modal-hero-bg'); + if (headerBg) { + headerBg.style.backgroundImage = `url('${data.image_url}')`; + } + + // Update avatar + const avatarContainer = modal.querySelector('.artist-download-modal-hero-avatar'); + if (avatarContainer) { + avatarContainer.innerHTML = `${artistBubbleData.artist.name}`; + } + + // Update artist object in memory + artistBubbleData.artist.image_url = data.image_url; + } + }) + .catch(err => console.error('❌ Failed to load modal image:', err)); + } } /**