Throttle Spotify pagination and harden watchlist scanner against rate limits

- Add rate limiting to all 4 Spotify pagination loops (get_artist_albums,
  get_user_playlists, get_playlist_tracks, get_album_tracks) — these
  called sp.next() bypassing the rate_limited decorator entirely, causing
  unthrottled API calls that triggered 429 bans
- Track pagination calls in API rate monitor (separate endpoint names)
- Increase DELAY_BETWEEN_ARTISTS from 2s to 4s in watchlist scanner
- Abort watchlist scan immediately if Spotify rate limit detected mid-scan
  instead of continuing to hammer the API

core/spotify_client.py

@@ -662,8 +662,18 @@ class SpotifyClient:
                     playlist = Playlist.from_spotify_playlist(playlist_data, tracks)
                     playlists.append(playlist)
                 
-                results = self.sp.next(results) if results['next'] else None
-            
+                if results['next']:
+                    with _api_call_lock:
+                        elapsed = time.time() - _last_api_call_time
+                        if elapsed < MIN_API_INTERVAL:
+                            time.sleep(MIN_API_INTERVAL - elapsed)
+                        globals()['_last_api_call_time'] = time.time()
+                    from core.api_call_tracker import api_call_tracker
+                    api_call_tracker.record_call('spotify', endpoint='get_user_playlists_page')
+                    results = self.sp.next(results)
+                else:
+                    results = None
+
             logger.info(f"Retrieved {len(playlists)} playlists")
             return playlists
             
@@ -931,7 +941,17 @@ class SpotifyClient:
                         track = Track.from_spotify_track(track_data)
                         tracks.append(track)
 
-                results = self.sp.next(results) if results['next'] else None
+                if results['next']:
+                    with _api_call_lock:
+                        elapsed = time.time() - _last_api_call_time
+                        if elapsed < MIN_API_INTERVAL:
+                            time.sleep(MIN_API_INTERVAL - elapsed)
+                        globals()['_last_api_call_time'] = time.time()
+                    from core.api_call_tracker import api_call_tracker
+                    api_call_tracker.record_call('spotify', endpoint='get_playlist_tracks_page')
+                    results = self.sp.next(results)
+                else:
+                    results = None
 
             return tracks
 
@@ -1263,9 +1283,16 @@ class SpotifyClient:
                 # Collect all tracks starting with first page
                 all_tracks = first_page['items'][:]
 
-                # Fetch remaining pages if they exist
+                # Fetch remaining pages if they exist — throttle pagination
                 next_page = first_page
                 while next_page.get('next'):
+                    with _api_call_lock:
+                        elapsed = time.time() - _last_api_call_time
+                        if elapsed < MIN_API_INTERVAL:
+                            time.sleep(MIN_API_INTERVAL - elapsed)
+                        globals()['_last_api_call_time'] = time.time()
+                    from core.api_call_tracker import api_call_tracker
+                    api_call_tracker.record_call('spotify', endpoint='get_album_tracks_page')
                     next_page = self.sp.next(next_page)
                     if next_page and 'items' in next_page:
                         all_tracks.extend(next_page['items'])
@@ -1338,8 +1365,19 @@ class SpotifyClient:
                         albums.append(album)
                         raw_items.append(album_data)
 
-                    # Get next batch if available
-                    results = self.sp.next(results) if results['next'] else None
+                    # Get next batch if available — throttle pagination to respect rate limits
+                    if results['next']:
+                        # Enforce same rate limit as decorated calls
+                        with _api_call_lock:
+                            elapsed = time.time() - _last_api_call_time
+                            if elapsed < MIN_API_INTERVAL:
+                                time.sleep(MIN_API_INTERVAL - elapsed)
+                            globals()['_last_api_call_time'] = time.time()
+                        from core.api_call_tracker import api_call_tracker
+                        api_call_tracker.record_call('spotify', endpoint='get_artist_albums_page')
+                        results = self.sp.next(results)
+                    else:
+                        results = None
 
                 logger.info(f"Retrieved {len(albums)} albums for artist {artist_id}")
 

core/watchlist_scanner.py

@@ -20,7 +20,7 @@ from utils.logging_config import get_logger
 logger = get_logger("watchlist_scanner")
 
 # Rate limiting constants for watchlist operations
-DELAY_BETWEEN_ARTISTS = 2.0      # 2 seconds between different artists
+DELAY_BETWEEN_ARTISTS = 4.0      # 4 seconds between different artists (was 2s, increased to reduce Spotify rate limit risk)
 DELAY_BETWEEN_ALBUMS = 0.5       # 500ms between albums for same artist
 DELAY_BETWEEN_API_BATCHES = 1.0  # 1 second between API batch operations
 
@@ -621,15 +621,20 @@ class WatchlistScanner:
             
             scan_results = []
             for i, artist in enumerate(watchlist_artists):
+                # Abort scan if Spotify is rate limited — don't keep hammering
+                if self.spotify_client and hasattr(self.spotify_client, 'is_rate_limited') and self.spotify_client.is_rate_limited():
+                    logger.warning(f"⚠️ Spotify rate limited — aborting watchlist scan after {i}/{len(watchlist_artists)} artists")
+                    break
+
                 try:
                     result = self.scan_artist(artist)
                     scan_results.append(result)
-                    
+
                     if result.success:
                         logger.info(f"✅ Scanned {artist.artist_name}: {result.new_tracks_found} new tracks found")
                     else:
                         logger.warning(f"❌ Failed to scan {artist.artist_name}: {result.error_message}")
-                    
+
                     # Rate limiting: Add delay between artists to avoid hitting Spotify API limits
                     # This is critical to prevent getting banned for 6+ hours
                     if i < len(watchlist_artists) - 1:  # Don't delay after the last artist