Discover page itunes integration. Spotify and Itunes will have their own pool

core/itunes_client.py

@@ -371,8 +371,13 @@ class iTunesClient:
 
         return albums[:limit]
     
-    def get_album(self, album_id: str) -> Optional[Dict[str, Any]]:
+    def get_album(self, album_id: str, include_tracks: bool = True) -> Optional[Dict[str, Any]]:
-        """Get album information - normalized to Spotify format"""
+        """Get album information with tracks - normalized to Spotify format.
+
+        Args:
+            album_id: iTunes album/collection ID
+            include_tracks: If True, also fetches and includes tracks (default True for Spotify compatibility)
+        """
         results = self._lookup(id=album_id)
 
         for album_data in results:
@@ -400,7 +405,7 @@ class iTunesClient:
                 else:
                     album_type = 'album'
 
-                return {
+                album_result = {
                     'id': str(album_data.get('collectionId', '')),
                     'name': album_data.get('collectionName', ''),
                     'images': images,
@@ -414,6 +419,16 @@ class iTunesClient:
                     '_raw_data': album_data
                 }
 
+                # Include tracks to match Spotify's get_album format
+                if include_tracks:
+                    tracks_data = self.get_album_tracks(album_id)
+                    if tracks_data and 'items' in tracks_data:
+                        album_result['tracks'] = tracks_data
+                    else:
+                        album_result['tracks'] = {'items': [], 'total': 0}
+
+                return album_result
+
         return None
     
     def get_album_tracks(self, album_id: str) -> Optional[Dict[str, Any]]:

core/metadata_service.py

@@ -43,7 +43,7 @@ class MetadataService:
     
     def _log_initialization(self):
         """Log initialization status"""
-        spotify_status = "✅ Authenticated" if self.spotify.is_authenticated() else "❌ Not authenticated"
+        spotify_status = "✅ Authenticated" if self.spotify.is_spotify_authenticated() else "❌ Not authenticated"
         itunes_status = "✅ Available" if self.itunes.is_authenticated() else "❌ Not available"
         
         logger.info(f"MetadataService initialized - Spotify: {spotify_status}, iTunes: {itunes_status}")
@@ -52,7 +52,7 @@ class MetadataService:
     def get_active_provider(self) -> str:
         """
         Get the currently active metadata provider.
-        
+
         Returns:
             "spotify" or "itunes"
         """
@@ -61,14 +61,16 @@ class MetadataService:
         elif self.preferred_provider == "itunes":
             return "itunes"
         else:  # auto
-            return "spotify" if self.spotify.is_authenticated() else "itunes"
+            # Use is_spotify_authenticated() to check actual Spotify auth status
+            # (is_authenticated() always returns True due to iTunes fallback)
+            return "spotify" if self.spotify.is_spotify_authenticated() else "itunes"
     
     def _get_client(self):
         """Get the appropriate client based on provider selection"""
         provider = self.get_active_provider()
-        
+
         if provider == "spotify":
-            if not self.spotify.is_authenticated():
+            if not self.spotify.is_spotify_authenticated():
                 logger.warning("Spotify requested but not authenticated, falling back to iTunes")
                 return self.itunes
             return self.spotify
@@ -168,38 +170,38 @@ class MetadataService:
     
     def get_user_playlists(self) -> List:
         """Get user playlists (Spotify only)"""
-        if self.get_active_provider() == "spotify" and self.spotify.is_authenticated():
+        if self.spotify.is_spotify_authenticated():
             return self.spotify.get_user_playlists()
         logger.warning("User playlists only available with Spotify authentication")
         return []
-    
+
     def get_saved_tracks(self) -> List:
         """Get user's saved/liked tracks (Spotify only)"""
-        if self.get_active_provider() == "spotify" and self.spotify.is_authenticated():
+        if self.spotify.is_spotify_authenticated():
             return self.spotify.get_saved_tracks()
         logger.warning("Saved tracks only available with Spotify authentication")
         return []
-    
+
     def get_saved_tracks_count(self) -> int:
         """Get count of user's saved tracks (Spotify only)"""
-        if self.get_active_provider() == "spotify" and self.spotify.is_authenticated():
+        if self.spotify.is_spotify_authenticated():
             return self.spotify.get_saved_tracks_count()
         return 0
-    
+
     # ==================== Utility Methods ====================
-    
+
     def is_authenticated(self) -> bool:
         """Check if any provider is available"""
-        return self.spotify.is_authenticated() or self.itunes.is_authenticated()
+        return self.spotify.is_spotify_authenticated() or self.itunes.is_authenticated()
-    
+
     def get_provider_info(self) -> Dict[str, Any]:
         """Get information about available providers"""
         return {
             "active_provider": self.get_active_provider(),
-            "spotify_authenticated": self.spotify.is_authenticated(),
+            "spotify_authenticated": self.spotify.is_spotify_authenticated(),
             "itunes_available": self.itunes.is_authenticated(),
             "preferred_provider": self.preferred_provider,
-            "can_access_user_data": self.spotify.is_authenticated(),
+            "can_access_user_data": self.spotify.is_spotify_authenticated(),
         }
     
     def reload_config(self):

core/personalized_playlists.py

@@ -112,7 +112,11 @@ class PersonalizedPlaylistsService:
 
     def _build_track_dict(self, row, source: str) -> Dict:
         """Build a standardized track dictionary from a database row."""
-        track_data = row['track_data_json']
+        # Convert sqlite3.Row to dict if needed (Row objects don't support .get())
+        if hasattr(row, 'keys'):
+            row = dict(row)
+
+        track_data = row.get('track_data_json')
         if isinstance(track_data, str):
             try:
                 track_data = json.loads(track_data)
@@ -123,11 +127,11 @@ class PersonalizedPlaylistsService:
             'track_id': row.get('spotify_track_id') or row.get('itunes_track_id'),
             'spotify_track_id': row.get('spotify_track_id'),
             'itunes_track_id': row.get('itunes_track_id'),
-            'track_name': row['track_name'],
+            'track_name': row.get('track_name', 'Unknown'),
-            'artist_name': row['artist_name'],
+            'artist_name': row.get('artist_name', 'Unknown'),
-            'album_name': row['album_name'],
+            'album_name': row.get('album_name', 'Unknown'),
-            'album_cover_url': row['album_cover_url'],
+            'album_cover_url': row.get('album_cover_url'),
-            'duration_ms': row['duration_ms'],
+            'duration_ms': row.get('duration_ms', 0),
             'popularity': row.get('popularity', 0),
             'track_data_json': track_data,
             'source': source

core/spotify_client.py

@@ -172,6 +172,19 @@ class SpotifyClient:
         self._itunes_client = None  # Lazy-loaded iTunes fallback
         self._setup_client()
 
+    def _is_spotify_id(self, id_str: str) -> bool:
+        """Check if an ID is a Spotify ID (alphanumeric) vs iTunes ID (numeric only)"""
+        if not id_str:
+            return False
+        # Spotify IDs contain letters and numbers, iTunes IDs are purely numeric
+        return not id_str.isdigit()
+
+    def _is_itunes_id(self, id_str: str) -> bool:
+        """Check if an ID is an iTunes ID (numeric only)"""
+        if not id_str:
+            return False
+        return id_str.isdigit()
+
     @property
     def _itunes(self):
         """Lazy-load iTunes client for fallback when Spotify not authenticated"""
@@ -534,9 +547,13 @@ class SpotifyClient:
                 logger.error(f"Error fetching track details via Spotify: {e}")
                 # Fall through to iTunes fallback
 
-        # iTunes fallback
+        # iTunes fallback - only if ID is numeric (iTunes format)
-        logger.debug(f"Using iTunes fallback for track details: {track_id}")
+        if self._is_itunes_id(track_id):
-        return self._itunes.get_track_details(track_id)
+            logger.debug(f"Using iTunes fallback for track details: {track_id}")
+            return self._itunes.get_track_details(track_id)
+        else:
+            logger.debug(f"Cannot use iTunes fallback for Spotify track ID: {track_id}")
+            return None
     
     @rate_limited
     def get_track_features(self, track_id: str) -> Optional[Dict[str, Any]]:
@@ -563,9 +580,13 @@ class SpotifyClient:
                 logger.error(f"Error fetching album via Spotify: {e}")
                 # Fall through to iTunes fallback
 
-        # iTunes fallback
+        # iTunes fallback - only if ID is numeric (iTunes format)
-        logger.debug(f"Using iTunes fallback for album: {album_id}")
+        if self._is_itunes_id(album_id):
-        return self._itunes.get_album(album_id)
+            logger.debug(f"Using iTunes fallback for album: {album_id}")
+            return self._itunes.get_album(album_id)
+        else:
+            logger.debug(f"Cannot use iTunes fallback for Spotify album ID: {album_id}")
+            return None
     
     @rate_limited
     def get_album_tracks(self, album_id: str) -> Optional[Dict[str, Any]]:
@@ -602,9 +623,13 @@ class SpotifyClient:
                 logger.error(f"Error fetching album tracks via Spotify: {e}")
                 # Fall through to iTunes fallback
 
-        # iTunes fallback
+        # iTunes fallback - only if ID is numeric (iTunes format)
-        logger.debug(f"Using iTunes fallback for album tracks: {album_id}")
+        if self._is_itunes_id(album_id):
-        return self._itunes.get_album_tracks(album_id)
+            logger.debug(f"Using iTunes fallback for album tracks: {album_id}")
+            return self._itunes.get_album_tracks(album_id)
+        else:
+            logger.debug(f"Cannot use iTunes fallback for Spotify album ID: {album_id}")
+            return None
     
     @rate_limited
     def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 50) -> List[Album]:
@@ -629,9 +654,13 @@ class SpotifyClient:
                 logger.error(f"Error fetching artist albums via Spotify: {e}")
                 # Fall through to iTunes fallback
 
-        # iTunes fallback
+        # iTunes fallback - only if ID is numeric (iTunes format)
-        logger.debug(f"Using iTunes fallback for artist albums: {artist_id}")
+        if self._is_itunes_id(artist_id):
-        return self._itunes.get_artist_albums(artist_id, album_type, limit)
+            logger.debug(f"Using iTunes fallback for artist albums: {artist_id}")
+            return self._itunes.get_artist_albums(artist_id, album_type, limit)
+        else:
+            logger.debug(f"Cannot use iTunes fallback for Spotify artist ID: {artist_id}")
+            return []
 
     @rate_limited
     def get_user_info(self) -> Optional[Dict[str, Any]]:
@@ -662,6 +691,10 @@ class SpotifyClient:
                 logger.error(f"Error fetching artist via Spotify: {e}")
                 # Fall through to iTunes fallback
 
-        # iTunes fallback
+        # iTunes fallback - only if ID is numeric (iTunes format)
-        logger.debug(f"Using iTunes fallback for artist: {artist_id}")
+        if self._is_itunes_id(artist_id):
-        return self._itunes.get_artist(artist_id)
+            logger.debug(f"Using iTunes fallback for artist: {artist_id}")
+            return self._itunes.get_artist(artist_id)
+        else:
+            logger.debug(f"Cannot use iTunes fallback for Spotify artist ID: {artist_id}")
+            return None

core/watchlist_scanner.py

@@ -280,6 +280,7 @@ class WatchlistScanner:
     def _get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist):
         """
         Get the appropriate client and artist ID based on active provider.
+        If iTunes ID is missing, searches by artist name to find and cache it.
 
         Returns:
             Tuple of (client, artist_id, provider_name) or (None, None, None) if no valid ID
@@ -296,9 +297,80 @@ class WatchlistScanner:
             if watchlist_artist.itunes_artist_id:
                 return (self.metadata_service.itunes, watchlist_artist.itunes_artist_id, 'itunes')
             else:
-                logger.warning(f"No iTunes ID for {watchlist_artist.artist_name}, cannot scan with iTunes")
+                # No iTunes ID stored - search by name and cache it
-                return (None, None, None)
+                logger.info(f"No iTunes ID for {watchlist_artist.artist_name}, searching by name...")
-    
+                try:
+                    itunes_client = self.metadata_service.itunes
+                    search_results = itunes_client.search_artists(watchlist_artist.artist_name, limit=1)
+                    if search_results and len(search_results) > 0:
+                        itunes_id = search_results[0].id
+                        logger.info(f"Found iTunes ID {itunes_id} for {watchlist_artist.artist_name}")
+                        # Cache the iTunes ID in the database for future use
+                        self.database.update_watchlist_artist_itunes_id(
+                            watchlist_artist.spotify_artist_id or str(watchlist_artist.id),
+                            itunes_id
+                        )
+                        return (itunes_client, itunes_id, 'itunes')
+                    else:
+                        logger.warning(f"Could not find {watchlist_artist.artist_name} on iTunes")
+                        return (None, None, None)
+                except Exception as e:
+                    logger.error(f"Error searching iTunes for {watchlist_artist.artist_name}: {e}")
+                    return (None, None, None)
+
+    def get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist):
+        """
+        Public wrapper for _get_active_client_and_artist_id.
+        Gets the appropriate client and artist ID based on active provider.
+
+        Returns:
+            Tuple of (client, artist_id, provider_name) or (None, None, None) if no valid ID
+        """
+        return self._get_active_client_and_artist_id(watchlist_artist)
+
+    def get_artist_image_url(self, watchlist_artist: WatchlistArtist) -> Optional[str]:
+        """
+        Get artist image URL using the active provider.
+
+        Returns:
+            Image URL string or None if not available
+        """
+        client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist)
+        if not client or not artist_id:
+            return None
+
+        try:
+            artist_data = client.get_artist(artist_id)
+            if artist_data:
+                # Handle both Spotify and iTunes response formats
+                if 'images' in artist_data and artist_data['images']:
+                    return artist_data['images'][0].get('url')
+                elif 'image_url' in artist_data:
+                    return artist_data['image_url']
+        except Exception as e:
+            logger.debug(f"Could not fetch artist image for {watchlist_artist.artist_name}: {e}")
+
+        return None
+
+    def get_artist_discography_for_watchlist(self, watchlist_artist: WatchlistArtist, last_scan_timestamp: Optional[datetime] = None) -> Optional[List]:
+        """
+        Get artist's discography using the active provider, with proper ID resolution.
+        This is the provider-aware version of get_artist_discography.
+
+        Args:
+            watchlist_artist: WatchlistArtist object (has both spotify and itunes IDs)
+            last_scan_timestamp: Only return releases after this date (for incremental scans)
+
+        Returns:
+            List of albums or None on error
+        """
+        client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist)
+        if not client or not artist_id:
+            logger.warning(f"No valid client/ID for {watchlist_artist.artist_name}")
+            return None
+
+        return self._get_artist_discography_with_client(client, artist_id, last_scan_timestamp)
+
     def scan_all_watchlist_artists(self) -> List[ScanResult]:
         """
         Scan artists in the watchlist for new releases.
@@ -562,7 +634,9 @@ class WatchlistScanner:
             try:
                 # Check if we have fresh similar artists cached (< 30 days old)
                 if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30):
-                    logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping fetch")
+                    logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping MusicMap fetch")
+                    # Even if cached, backfill missing iTunes IDs (seamless dual-source support)
+                    self._backfill_similar_artists_itunes_ids(source_artist_id)
                 else:
                     logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}...")
                     self.update_similar_artists(watchlist_artist)
@@ -812,6 +886,10 @@ class WatchlistScanner:
                 album_date = datetime(int(year), int(month), 1, tzinfo=timezone.utc)
             elif len(release_date_str) == 10:  # Full date (e.g., "2023-10-15")
                 album_date = datetime.strptime(release_date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc)
+            elif 'T' in release_date_str:  # ISO 8601 with time (e.g., "2017-12-08T08:00:00Z" from iTunes)
+                # Strip the time portion and parse just the date
+                date_part = release_date_str.split('T')[0]
+                album_date = datetime.strptime(date_part, "%Y-%m-%d").replace(tzinfo=timezone.utc)
             else:
                 logger.warning(f"Unknown release date format: {release_date_str}")
                 return True  # Include if we can't parse
@@ -1246,6 +1324,54 @@ class WatchlistScanner:
             logger.error(f"Error fetching similar artists from MusicMap: {e}")
             return []
 
+    def _backfill_similar_artists_itunes_ids(self, source_artist_id: str) -> int:
+        """
+        Backfill missing iTunes IDs for cached similar artists.
+        This ensures seamless dual-source support without clearing cached data.
+
+        Args:
+            source_artist_id: The source artist ID to backfill similar artists for
+
+        Returns:
+            Number of similar artists updated with iTunes IDs
+        """
+        try:
+            # Get similar artists that are missing iTunes IDs
+            similar_artists = self.database.get_similar_artists_missing_itunes_ids(source_artist_id)
+
+            if not similar_artists:
+                return 0
+
+            logger.info(f"Backfilling iTunes IDs for {len(similar_artists)} similar artists")
+
+            # Get iTunes client
+            from core.itunes_client import iTunesClient
+            itunes_client = iTunesClient()
+
+            updated_count = 0
+            for similar_artist in similar_artists:
+                try:
+                    # Search iTunes by artist name
+                    itunes_results = itunes_client.search_artists(similar_artist.similar_artist_name, limit=1)
+                    if itunes_results and len(itunes_results) > 0:
+                        itunes_id = itunes_results[0].id
+                        # Update the similar artist with the iTunes ID
+                        if self.database.update_similar_artist_itunes_id(similar_artist.id, itunes_id):
+                            updated_count += 1
+                            logger.debug(f"  Backfilled iTunes ID {itunes_id} for {similar_artist.similar_artist_name}")
+                except Exception as e:
+                    logger.debug(f"  Could not backfill iTunes ID for {similar_artist.similar_artist_name}: {e}")
+                    continue
+
+            if updated_count > 0:
+                logger.info(f"Backfilled {updated_count} similar artists with iTunes IDs")
+
+            return updated_count
+
+        except Exception as e:
+            logger.error(f"Error backfilling similar artists iTunes IDs: {e}")
+            return 0
+
     def update_similar_artists(self, watchlist_artist: WatchlistArtist, limit: int = 10) -> bool:
         """
         Fetch and store similar artists for a watchlist artist.
@@ -1351,8 +1477,21 @@ class WatchlistScanner:
                     sources_to_process = []
 
                     # Always add iTunes first (baseline source)
-                    if similar_artist.similar_artist_itunes_id:
+                    itunes_id = similar_artist.similar_artist_itunes_id
-                        sources_to_process.append(('itunes', similar_artist.similar_artist_itunes_id))
+                    if not itunes_id:
+                        # On-the-fly lookup for missing iTunes ID (seamless provider switching)
+                        try:
+                            itunes_results = itunes_client.search_artists(similar_artist.similar_artist_name, limit=1)
+                            if itunes_results and len(itunes_results) > 0:
+                                itunes_id = itunes_results[0].id
+                                # Cache it for future use
+                                self.database.update_similar_artist_itunes_id(similar_artist.id, itunes_id)
+                                logger.debug(f"  Resolved iTunes ID {itunes_id} for {similar_artist.similar_artist_name}")
+                        except Exception as e:
+                            logger.debug(f"  Could not resolve iTunes ID for {similar_artist.similar_artist_name}: {e}")
+
+                    if itunes_id:
+                        sources_to_process.append(('itunes', itunes_id))
 
                     # Add Spotify if authenticated and we have an ID
                     if spotify_available and similar_artist.similar_artist_spotify_id:

database/music_database.py

@@ -591,29 +591,7 @@ class MusicDatabase:
                 )
             """)
 
-            # Create indexes for performance
+            # ============== MIGRATIONS (must run BEFORE index creation on new columns) ==============
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_source ON similar_artists (source_artist_id)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_spotify ON similar_artists (similar_artist_spotify_id)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_itunes ON similar_artists (similar_artist_itunes_id)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_occurrence ON similar_artists (occurrence_count)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_name ON similar_artists (similar_artist_name)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_spotify_track ON discovery_pool (spotify_track_id)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_itunes_track ON discovery_pool (itunes_track_id)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_artist ON discovery_pool (spotify_artist_id)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_itunes_artist ON discovery_pool (itunes_artist_id)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_source ON discovery_pool (source)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_added_date ON discovery_pool (added_date)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_is_new ON discovery_pool (is_new_release)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_watchlist ON recent_releases (watchlist_artist_id)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_date ON recent_releases (release_date)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_source ON recent_releases (source)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_source ON discovery_recent_albums (source)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_date ON discovery_recent_albums (release_date)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_playlists_type ON listenbrainz_playlists (playlist_type)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_playlists_mbid ON listenbrainz_playlists (playlist_mbid)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_tracks_playlist ON listenbrainz_tracks (playlist_id)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_tracks_position ON listenbrainz_tracks (playlist_id, position)")
-            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_artist ON discovery_recent_albums (artist_spotify_id)")
 
             # Add genres column to discovery_pool if it doesn't exist (migration)
             cursor.execute("PRAGMA table_info(discovery_pool)")
@@ -658,6 +636,30 @@ class MusicDatabase:
                 cursor.execute("ALTER TABLE discovery_recent_albums ADD COLUMN source TEXT DEFAULT 'spotify'")
                 logger.info("Added iTunes columns to discovery_recent_albums table for dual-source discovery")
 
+            # ============== INDEXES (after migrations to ensure columns exist) ==============
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_source ON similar_artists (source_artist_id)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_spotify ON similar_artists (similar_artist_spotify_id)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_itunes ON similar_artists (similar_artist_itunes_id)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_occurrence ON similar_artists (occurrence_count)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_name ON similar_artists (similar_artist_name)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_spotify_track ON discovery_pool (spotify_track_id)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_itunes_track ON discovery_pool (itunes_track_id)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_artist ON discovery_pool (spotify_artist_id)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_itunes_artist ON discovery_pool (itunes_artist_id)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_source ON discovery_pool (source)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_added_date ON discovery_pool (added_date)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_is_new ON discovery_pool (is_new_release)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_watchlist ON recent_releases (watchlist_artist_id)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_date ON recent_releases (release_date)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_source ON recent_releases (source)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_source ON discovery_recent_albums (source)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_date ON discovery_recent_albums (release_date)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_playlists_type ON listenbrainz_playlists (playlist_type)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_playlists_mbid ON listenbrainz_playlists (playlist_mbid)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_tracks_playlist ON listenbrainz_tracks (playlist_id)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_listenbrainz_tracks_position ON listenbrainz_tracks (playlist_id, position)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_artist ON discovery_recent_albums (artist_spotify_id)")
+
             logger.info("Discovery tables created successfully")
 
         except Exception as e:
@@ -2996,6 +2998,27 @@ class MusicDatabase:
             logger.error(f"Error updating watchlist iTunes ID: {e}")
             return False
 
+    def update_watchlist_artist_itunes_id(self, spotify_artist_id: str, itunes_id: str) -> bool:
+        """Update the iTunes artist ID for a watchlist artist by Spotify ID (for cross-provider caching)"""
+        try:
+            with self._get_connection() as conn:
+                cursor = conn.cursor()
+
+                cursor.execute("""
+                    UPDATE watchlist_artists
+                    SET itunes_artist_id = ?, updated_at = CURRENT_TIMESTAMP
+                    WHERE spotify_artist_id = ?
+                """, (itunes_id, spotify_artist_id))
+
+                conn.commit()
+                if cursor.rowcount > 0:
+                    logger.info(f"Cached iTunes ID {itunes_id} for Spotify artist {spotify_artist_id}")
+                return cursor.rowcount > 0
+
+        except Exception as e:
+            logger.error(f"Error caching watchlist iTunes ID: {e}")
+            return False
+
     # === Discovery Feature Methods ===
 
     def add_or_update_similar_artist(self, source_artist_id: str, similar_artist_name: str,
@@ -3056,10 +3079,66 @@ class MusicDatabase:
             logger.error(f"Error getting similar artists: {e}")
             return []
 
-    def has_fresh_similar_artists(self, source_artist_id: str, days_threshold: int = 30) -> bool:
+    def get_similar_artists_missing_itunes_ids(self, source_artist_id: str) -> List[SimilarArtist]:
+        """Get similar artists for a source that are missing iTunes IDs (for backfill)"""
+        try:
+            with self._get_connection() as conn:
+                cursor = conn.cursor()
+
+                cursor.execute("""
+                    SELECT * FROM similar_artists
+                    WHERE source_artist_id = ?
+                    AND (similar_artist_itunes_id IS NULL OR similar_artist_itunes_id = '')
+                    ORDER BY occurrence_count DESC
+                    LIMIT 50
+                """, (source_artist_id,))
+
+                rows = cursor.fetchall()
+                return [SimilarArtist(
+                    id=row['id'],
+                    source_artist_id=row['source_artist_id'],
+                    similar_artist_spotify_id=row['similar_artist_spotify_id'],
+                    similar_artist_itunes_id=None,
+                    similar_artist_name=row['similar_artist_name'],
+                    similarity_rank=row['similarity_rank'],
+                    occurrence_count=row['occurrence_count'],
+                    last_updated=datetime.fromisoformat(row['last_updated'])
+                ) for row in rows]
+
+        except Exception as e:
+            logger.error(f"Error getting similar artists missing iTunes IDs: {e}")
+            return []
+
+    def update_similar_artist_itunes_id(self, similar_artist_id: int, itunes_id: str) -> bool:
+        """Update a similar artist's iTunes ID (for backfill)"""
+        try:
+            with self._get_connection() as conn:
+                cursor = conn.cursor()
+
+                cursor.execute("""
+                    UPDATE similar_artists
+                    SET similar_artist_itunes_id = ?
+                    WHERE id = ?
+                """, (itunes_id, similar_artist_id))
+
+                conn.commit()
+                return cursor.rowcount > 0
+
+        except Exception as e:
+            logger.error(f"Error updating similar artist iTunes ID: {e}")
+            return False
+
+    def has_fresh_similar_artists(self, source_artist_id: str, days_threshold: int = 30, require_itunes: bool = True) -> bool:
         """
         Check if we have cached similar artists that are still fresh (< days_threshold old).
-        Returns True if we have recent data, False if data is stale or missing.
+        Also checks that similar artists have the required provider IDs.
+
+        Args:
+            source_artist_id: The source artist ID to check
+            days_threshold: Maximum age in days to consider fresh
+            require_itunes: If True, also requires iTunes IDs to be present (for seamless provider switching)
+
+        Returns True if we have recent data with required IDs, False if data is stale, missing, or incomplete.
         """
         try:
             with self._get_connection() as conn:
@@ -3081,7 +3160,27 @@ class MusicDatabase:
                 last_updated = datetime.fromisoformat(row['last_updated'])
                 days_since_update = (datetime.now() - last_updated).total_seconds() / 86400  # seconds to days
 
-                return days_since_update < days_threshold
+                if days_since_update >= days_threshold:
+                    return False
+
+                # Check if we have iTunes IDs (for seamless provider switching)
+                if require_itunes:
+                    cursor.execute("""
+                        SELECT COUNT(*) as total,
+                               SUM(CASE WHEN similar_artist_itunes_id IS NOT NULL AND similar_artist_itunes_id != '' THEN 1 ELSE 0 END) as has_itunes
+                        FROM similar_artists
+                        WHERE source_artist_id = ?
+                    """, (source_artist_id,))
+                    id_row = cursor.fetchone()
+
+                    if id_row and id_row['total'] > 0:
+                        # If less than 50% have iTunes IDs, consider stale and refetch
+                        itunes_ratio = id_row['has_itunes'] / id_row['total']
+                        if itunes_ratio < 0.5:
+                            logger.debug(f"Similar artists for {source_artist_id} missing iTunes IDs ({id_row['has_itunes']}/{id_row['total']}), will refetch")
+                            return False
+
+                return True
 
         except Exception as e:
             logger.error(f"Error checking similar artists freshness: {e}")

web_server.py

@@ -17269,14 +17269,8 @@ def start_watchlist_scan():
                 
                 for i, artist in enumerate(watchlist_artists):
                     try:
-                        # Fetch artist image
+                        # Fetch artist image using provider-aware method
-                        artist_image_url = ''
+                        artist_image_url = scanner.get_artist_image_url(artist) or ''
-                        try:
-                            artist_data = spotify_client.get_artist(artist.spotify_artist_id)
-                            if artist_data and 'images' in artist_data and artist_data['images']:
-                                artist_image_url = artist_data['images'][0]['url']
-                        except:
-                            pass
 
                         # Update progress
                         watchlist_scan_state.update({
@@ -17290,9 +17284,9 @@ def start_watchlist_scan():
                             'current_album_image_url': '',
                             'current_track_name': ''
                         })
-                        
+
-                        # Get artist discography
+                        # Get artist discography using provider-aware method
-                        albums = scanner.get_artist_discography(artist.spotify_artist_id, artist.last_scan_timestamp)
+                        albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp)
                         
                         if albums is None:
                             scan_results.append(type('ScanResult', (), {
@@ -17320,8 +17314,8 @@ def start_watchlist_scan():
                         # Scan each album
                         for album_index, album in enumerate(albums):
                             try:
-                                # Get album tracks
+                                # Get album tracks using provider-aware method
-                                album_data = scanner.spotify_client.get_album(album.id)
+                                album_data = scanner.metadata_service.get_album(album.id)
                                 if not album_data or 'tracks' not in album_data:
                                     continue
 
@@ -17969,14 +17963,8 @@ def _process_watchlist_scan_automatically():
             # Scan each artist with detailed tracking
             for i, artist in enumerate(watchlist_artists):
                 try:
-                    # Fetch artist image
+                    # Fetch artist image using provider-aware method
-                    artist_image_url = ''
+                    artist_image_url = scanner.get_artist_image_url(artist) or ''
-                    try:
-                        artist_data = spotify_client.get_artist(artist.spotify_artist_id)
-                        if artist_data and 'images' in artist_data and artist_data['images']:
-                            artist_image_url = artist_data['images'][0]['url']
-                    except:
-                        pass
 
                     # Update progress
                     watchlist_scan_state.update({
@@ -17991,8 +17979,8 @@ def _process_watchlist_scan_automatically():
                         'current_track_name': ''
                     })
 
-                    # Get artist discography
+                    # Get artist discography using provider-aware method
-                    albums = scanner.get_artist_discography(artist.spotify_artist_id, artist.last_scan_timestamp)
+                    albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp)
 
                     if albums is None:
                         scan_results.append(type('ScanResult', (), {
@@ -18020,8 +18008,8 @@ def _process_watchlist_scan_automatically():
                     # Scan each album
                     for album_index, album in enumerate(albums):
                         try:
-                            # Get album tracks
+                            # Get album tracks using provider-aware method
-                            album_data = scanner.spotify_client.get_album(album.id)
+                            album_data = scanner.metadata_service.get_album(album.id)
                             if not album_data or 'tracks' not in album_data:
                                 continue