From 308f0f9711c94a54148083966b213648111591b5 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Tue, 17 Feb 2026 16:45:09 -0800 Subject: [PATCH] Add retry logic and adaptive rate limiting to watchlist scan Spotify's @rate_limited decorator now retries on 429/5xx with exponential backoff (up to 5 retries) instead of sleeping once and raising. Watchlist scan delays scale dynamically based on lookback setting and artist count to prevent sustained API pressure. A circuit breaker pauses the scan after consecutive rate-limit failures. --- core/spotify_client.py | 73 ++++++++++++++++--------- web_server.py | 118 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 155 insertions(+), 36 deletions(-) diff --git a/core/spotify_client.py b/core/spotify_client.py index e96beec7..1f2a613f 100644 --- a/core/spotify_client.py +++ b/core/spotify_client.py @@ -21,34 +21,57 @@ _request_queue = queue.Queue() _queue_processor_running = False def rate_limited(func): - """Decorator to enforce rate limiting on Spotify API calls""" + """Decorator to enforce rate limiting on Spotify API calls with retry and exponential backoff""" @wraps(func) def wrapper(*args, **kwargs): global _last_api_call_time - - with _api_call_lock: - current_time = time.time() - time_since_last_call = current_time - _last_api_call_time - - if time_since_last_call < MIN_API_INTERVAL: - sleep_time = MIN_API_INTERVAL - time_since_last_call - time.sleep(sleep_time) - - _last_api_call_time = time.time() - - try: - result = func(*args, **kwargs) - return result - except Exception as e: - # Implement exponential backoff for API errors - if "rate limit" in str(e).lower() or "429" in str(e): - logger.warning(f"Rate limit hit, implementing backoff: {e}") - # Use longer backoff to avoid getting banned - time.sleep(3.0) # Wait 3 seconds before retrying - elif "503" in str(e) or "502" in str(e): - logger.warning(f"Spotify service error, backing off: {e}") - time.sleep(2.0) # Wait 2 seconds for service errors - raise e + + max_retries = 5 + + for attempt in range(max_retries + 1): + # Enforce minimum interval between API calls + with _api_call_lock: + current_time = time.time() + time_since_last_call = current_time - _last_api_call_time + + if time_since_last_call < MIN_API_INTERVAL: + sleep_time = MIN_API_INTERVAL - time_since_last_call + time.sleep(sleep_time) + + _last_api_call_time = time.time() + + try: + return func(*args, **kwargs) + except Exception as e: + error_str = str(e).lower() + is_rate_limit = "rate limit" in error_str or "429" in str(e) + is_server_error = "502" in str(e) or "503" in str(e) + + if is_rate_limit and attempt < max_retries: + # Try to extract Retry-After from spotipy exception headers + retry_after = None + if hasattr(e, 'headers') and e.headers: + retry_after = e.headers.get('Retry-After') or e.headers.get('retry-after') + + if retry_after: + try: + delay = int(retry_after) + 1 + except (ValueError, TypeError): + delay = 3.0 * (2 ** attempt) + else: + delay = 3.0 * (2 ** attempt) # 3, 6, 12, 24, 48 + + logger.warning(f"Spotify rate limit hit, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries}): {func.__name__}") + time.sleep(delay) + continue + + elif is_server_error and attempt < max_retries: + delay = 2.0 * (2 ** attempt) # 2, 4, 8, 16, 32 + logger.warning(f"Spotify server error, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries}): {func.__name__}") + time.sleep(delay) + continue + + raise return wrapper @dataclass diff --git a/web_server.py b/web_server.py index 13b88ae9..b1b2a794 100644 --- a/web_server.py +++ b/web_server.py @@ -19239,7 +19239,37 @@ def start_watchlist_scan(): }) scan_results = [] - + + # Dynamic delay calculation based on scan scope + lookback_period = scanner._get_lookback_period_setting() + is_full_discography = (lookback_period == 'all') + artist_count = len(watchlist_artists) + + base_artist_delay = 2.0 + base_album_delay = 0.5 + + # Scale up for full discography (way more albums per artist) + if is_full_discography: + base_artist_delay *= 2.0 + base_album_delay *= 2.0 + + # Scale up further for large artist counts (sustained API pressure) + if artist_count > 200: + base_artist_delay *= 1.5 + base_album_delay *= 1.25 + elif artist_count > 100: + base_artist_delay *= 1.25 + + artist_delay = base_artist_delay + album_delay = base_album_delay + print(f"📊 Scan parameters: {artist_count} artists, lookback={lookback_period}, " + f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album") + + # Circuit breaker: pause scan on consecutive rate-limit failures + consecutive_failures = 0 + CIRCUIT_BREAKER_THRESHOLD = 3 + circuit_breaker_pause = 60 # seconds, doubles each trigger, max 600s + for i, artist in enumerate(watchlist_artists): try: # Fetch artist image using provider-aware method @@ -19335,14 +19365,14 @@ def start_watchlist_scan(): if len(watchlist_scan_state['recent_wishlist_additions']) > 10: watchlist_scan_state['recent_wishlist_additions'].pop() - # Small delay between albums + # Rate-limited delay between albums import time - time.sleep(0.5) - + time.sleep(album_delay) + except Exception as e: print(f"Error checking album {album.name}: {e}") continue - + # Update scan timestamp scanner.update_artist_scan_timestamp(artist) @@ -19381,10 +19411,28 @@ def start_watchlist_scan(): # Delay between artists if i < len(watchlist_artists) - 1: watchlist_scan_state['current_phase'] = 'rate_limiting' - time.sleep(2.0) - + time.sleep(artist_delay) + + # Reset circuit breaker on successful artist scan + consecutive_failures = 0 + circuit_breaker_pause = 60 + except Exception as e: print(f"Error scanning artist {artist.artist_name}: {e}") + + # Circuit breaker: detect consecutive rate-limit failures + error_str = str(e).lower() + if "429" in error_str or "rate limit" in error_str: + consecutive_failures += 1 + if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD: + print(f"🛑 Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s") + watchlist_scan_state['current_phase'] = 'circuit_breaker_pause' + time.sleep(circuit_breaker_pause) + circuit_breaker_pause = min(circuit_breaker_pause * 2, 600) + consecutive_failures = 0 + else: + consecutive_failures = 0 + scan_results.append(type('ScanResult', (), { 'artist_name': artist.artist_name, 'spotify_artist_id': artist.spotify_artist_id, @@ -19394,7 +19442,7 @@ def start_watchlist_scan(): 'success': False, 'error_message': str(e) })()) - + # Store final results watchlist_scan_state['status'] = 'completed' watchlist_scan_state['results'] = scan_results @@ -19957,6 +20005,36 @@ def _process_watchlist_scan_automatically(): scan_results = [] + # Dynamic delay calculation based on scan scope + lookback_period = scanner._get_lookback_period_setting() + is_full_discography = (lookback_period == 'all') + artist_count = len(watchlist_artists) + + base_artist_delay = 2.0 + base_album_delay = 0.5 + + # Scale up for full discography (way more albums per artist) + if is_full_discography: + base_artist_delay *= 2.0 + base_album_delay *= 2.0 + + # Scale up further for large artist counts (sustained API pressure) + if artist_count > 200: + base_artist_delay *= 1.5 + base_album_delay *= 1.25 + elif artist_count > 100: + base_artist_delay *= 1.25 + + artist_delay = base_artist_delay + album_delay = base_album_delay + print(f"📊 [Auto-Watchlist] Scan parameters: {artist_count} artists, lookback={lookback_period}, " + f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album") + + # Circuit breaker: pause scan on consecutive rate-limit failures + consecutive_failures = 0 + CIRCUIT_BREAKER_THRESHOLD = 3 + circuit_breaker_pause = 60 # seconds, doubles each trigger, max 600s + # Scan each artist with detailed tracking for i, artist in enumerate(watchlist_artists): try: @@ -20053,9 +20131,9 @@ def _process_watchlist_scan_automatically(): if len(watchlist_scan_state['recent_wishlist_additions']) > 10: watchlist_scan_state['recent_wishlist_additions'].pop() - # Small delay between albums + # Rate-limited delay between albums import time - time.sleep(0.5) + time.sleep(album_delay) except Exception as e: print(f"Error checking album {album.name}: {e}") @@ -20080,10 +20158,28 @@ def _process_watchlist_scan_automatically(): # Delay between artists if i < len(watchlist_artists) - 1: watchlist_scan_state['current_phase'] = 'rate_limiting' - time.sleep(2.0) + time.sleep(artist_delay) + + # Reset circuit breaker on successful artist scan + consecutive_failures = 0 + circuit_breaker_pause = 60 except Exception as e: print(f"Error scanning artist {artist.artist_name}: {e}") + + # Circuit breaker: detect consecutive rate-limit failures + error_str = str(e).lower() + if "429" in error_str or "rate limit" in error_str: + consecutive_failures += 1 + if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD: + print(f"🛑 [Auto-Watchlist] Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s") + watchlist_scan_state['current_phase'] = 'circuit_breaker_pause' + time.sleep(circuit_breaker_pause) + circuit_breaker_pause = min(circuit_breaker_pause * 2, 600) + consecutive_failures = 0 + else: + consecutive_failures = 0 + scan_results.append(type('ScanResult', (), { 'artist_name': artist.artist_name, 'spotify_artist_id': artist.spotify_artist_id,