Add retry logic and adaptive rate limiting to watchlist scan

Spotify's @rate_limited decorator now retries on 429/5xx with exponential backoff (up to 5 retries) instead of sleeping once and raising. Watchlist scan delays scale dynamically based on lookback setting and artist count to prevent sustained API pressure. A circuit breaker pauses the scan after consecutive rate-limit failures.

core/spotify_client.py

@@ -21,34 +21,57 @@ _request_queue = queue.Queue()
 _queue_processor_running = False
 
 def rate_limited(func):
-    """Decorator to enforce rate limiting on Spotify API calls"""
+    """Decorator to enforce rate limiting on Spotify API calls with retry and exponential backoff"""
     @wraps(func)
     def wrapper(*args, **kwargs):
         global _last_api_call_time
 
-        with _api_call_lock:
+        max_retries = 5
-            current_time = time.time()
-            time_since_last_call = current_time - _last_api_call_time
 
-            if time_since_last_call < MIN_API_INTERVAL:
+        for attempt in range(max_retries + 1):
-                sleep_time = MIN_API_INTERVAL - time_since_last_call
+            # Enforce minimum interval between API calls
-                time.sleep(sleep_time)
+            with _api_call_lock:
+                current_time = time.time()
+                time_since_last_call = current_time - _last_api_call_time
 
-            _last_api_call_time = time.time()
+                if time_since_last_call < MIN_API_INTERVAL:
+                    sleep_time = MIN_API_INTERVAL - time_since_last_call
+                    time.sleep(sleep_time)
 
-        try:
+                _last_api_call_time = time.time()
-            result = func(*args, **kwargs)
+
-            return result
+            try:
-        except Exception as e:
+                return func(*args, **kwargs)
-            # Implement exponential backoff for API errors
+            except Exception as e:
-            if "rate limit" in str(e).lower() or "429" in str(e):
+                error_str = str(e).lower()
-                logger.warning(f"Rate limit hit, implementing backoff: {e}")
+                is_rate_limit = "rate limit" in error_str or "429" in str(e)
-                # Use longer backoff to avoid getting banned
+                is_server_error = "502" in str(e) or "503" in str(e)
-                time.sleep(3.0)  # Wait 3 seconds before retrying
+
-            elif "503" in str(e) or "502" in str(e):
+                if is_rate_limit and attempt < max_retries:
-                logger.warning(f"Spotify service error, backing off: {e}")
+                    # Try to extract Retry-After from spotipy exception headers
-                time.sleep(2.0)  # Wait 2 seconds for service errors
+                    retry_after = None
-            raise e
+                    if hasattr(e, 'headers') and e.headers:
+                        retry_after = e.headers.get('Retry-After') or e.headers.get('retry-after')
+
+                    if retry_after:
+                        try:
+                            delay = int(retry_after) + 1
+                        except (ValueError, TypeError):
+                            delay = 3.0 * (2 ** attempt)
+                    else:
+                        delay = 3.0 * (2 ** attempt)  # 3, 6, 12, 24, 48
+
+                    logger.warning(f"Spotify rate limit hit, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries}): {func.__name__}")
+                    time.sleep(delay)
+                    continue
+
+                elif is_server_error and attempt < max_retries:
+                    delay = 2.0 * (2 ** attempt)  # 2, 4, 8, 16, 32
+                    logger.warning(f"Spotify server error, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries}): {func.__name__}")
+                    time.sleep(delay)
+                    continue
+
+                raise
     return wrapper
 
 @dataclass

web_server.py

@@ -19240,6 +19240,36 @@ def start_watchlist_scan():
                 
                 scan_results = []
 
+                # Dynamic delay calculation based on scan scope
+                lookback_period = scanner._get_lookback_period_setting()
+                is_full_discography = (lookback_period == 'all')
+                artist_count = len(watchlist_artists)
+
+                base_artist_delay = 2.0
+                base_album_delay = 0.5
+
+                # Scale up for full discography (way more albums per artist)
+                if is_full_discography:
+                    base_artist_delay *= 2.0
+                    base_album_delay *= 2.0
+
+                # Scale up further for large artist counts (sustained API pressure)
+                if artist_count > 200:
+                    base_artist_delay *= 1.5
+                    base_album_delay *= 1.25
+                elif artist_count > 100:
+                    base_artist_delay *= 1.25
+
+                artist_delay = base_artist_delay
+                album_delay = base_album_delay
+                print(f"📊 Scan parameters: {artist_count} artists, lookback={lookback_period}, "
+                      f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album")
+
+                # Circuit breaker: pause scan on consecutive rate-limit failures
+                consecutive_failures = 0
+                CIRCUIT_BREAKER_THRESHOLD = 3
+                circuit_breaker_pause = 60  # seconds, doubles each trigger, max 600s
+
                 for i, artist in enumerate(watchlist_artists):
                     try:
                         # Fetch artist image using provider-aware method
@@ -19335,9 +19365,9 @@ def start_watchlist_scan():
                                             if len(watchlist_scan_state['recent_wishlist_additions']) > 10:
                                                 watchlist_scan_state['recent_wishlist_additions'].pop()
                                 
-                                # Small delay between albums
+                                # Rate-limited delay between albums
                                 import time
-                                time.sleep(0.5)
+                                time.sleep(album_delay)
 
                             except Exception as e:
                                 print(f"Error checking album {album.name}: {e}")
@@ -19381,10 +19411,28 @@ def start_watchlist_scan():
                         # Delay between artists
                         if i < len(watchlist_artists) - 1:
                             watchlist_scan_state['current_phase'] = 'rate_limiting'
-                            time.sleep(2.0)
+                            time.sleep(artist_delay)
+
+                        # Reset circuit breaker on successful artist scan
+                        consecutive_failures = 0
+                        circuit_breaker_pause = 60
 
                     except Exception as e:
                         print(f"Error scanning artist {artist.artist_name}: {e}")
+
+                        # Circuit breaker: detect consecutive rate-limit failures
+                        error_str = str(e).lower()
+                        if "429" in error_str or "rate limit" in error_str:
+                            consecutive_failures += 1
+                            if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD:
+                                print(f"🛑 Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s")
+                                watchlist_scan_state['current_phase'] = 'circuit_breaker_pause'
+                                time.sleep(circuit_breaker_pause)
+                                circuit_breaker_pause = min(circuit_breaker_pause * 2, 600)
+                                consecutive_failures = 0
+                        else:
+                            consecutive_failures = 0
+
                         scan_results.append(type('ScanResult', (), {
                             'artist_name': artist.artist_name,
                             'spotify_artist_id': artist.spotify_artist_id,
@@ -19957,6 +20005,36 @@ def _process_watchlist_scan_automatically():
 
             scan_results = []
 
+            # Dynamic delay calculation based on scan scope
+            lookback_period = scanner._get_lookback_period_setting()
+            is_full_discography = (lookback_period == 'all')
+            artist_count = len(watchlist_artists)
+
+            base_artist_delay = 2.0
+            base_album_delay = 0.5
+
+            # Scale up for full discography (way more albums per artist)
+            if is_full_discography:
+                base_artist_delay *= 2.0
+                base_album_delay *= 2.0
+
+            # Scale up further for large artist counts (sustained API pressure)
+            if artist_count > 200:
+                base_artist_delay *= 1.5
+                base_album_delay *= 1.25
+            elif artist_count > 100:
+                base_artist_delay *= 1.25
+
+            artist_delay = base_artist_delay
+            album_delay = base_album_delay
+            print(f"📊 [Auto-Watchlist] Scan parameters: {artist_count} artists, lookback={lookback_period}, "
+                  f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album")
+
+            # Circuit breaker: pause scan on consecutive rate-limit failures
+            consecutive_failures = 0
+            CIRCUIT_BREAKER_THRESHOLD = 3
+            circuit_breaker_pause = 60  # seconds, doubles each trigger, max 600s
+
             # Scan each artist with detailed tracking
             for i, artist in enumerate(watchlist_artists):
                 try:
@@ -20053,9 +20131,9 @@ def _process_watchlist_scan_automatically():
                                         if len(watchlist_scan_state['recent_wishlist_additions']) > 10:
                                             watchlist_scan_state['recent_wishlist_additions'].pop()
 
-                            # Small delay between albums
+                            # Rate-limited delay between albums
                             import time
-                            time.sleep(0.5)
+                            time.sleep(album_delay)
 
                         except Exception as e:
                             print(f"Error checking album {album.name}: {e}")
@@ -20080,10 +20158,28 @@ def _process_watchlist_scan_automatically():
                     # Delay between artists
                     if i < len(watchlist_artists) - 1:
                         watchlist_scan_state['current_phase'] = 'rate_limiting'
-                        time.sleep(2.0)
+                        time.sleep(artist_delay)
+
+                    # Reset circuit breaker on successful artist scan
+                    consecutive_failures = 0
+                    circuit_breaker_pause = 60
 
                 except Exception as e:
                     print(f"Error scanning artist {artist.artist_name}: {e}")
+
+                    # Circuit breaker: detect consecutive rate-limit failures
+                    error_str = str(e).lower()
+                    if "429" in error_str or "rate limit" in error_str:
+                        consecutive_failures += 1
+                        if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD:
+                            print(f"🛑 [Auto-Watchlist] Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s")
+                            watchlist_scan_state['current_phase'] = 'circuit_breaker_pause'
+                            time.sleep(circuit_breaker_pause)
+                            circuit_breaker_pause = min(circuit_breaker_pause * 2, 600)
+                            consecutive_failures = 0
+                    else:
+                        consecutive_failures = 0
+
                     scan_results.append(type('ScanResult', (), {
                         'artist_name': artist.artist_name,
                         'spotify_artist_id': artist.spotify_artist_id,