danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Add retry logic and adaptive rate limiting to watchlist scan

Browse source 
Spotify's @rate_limited decorator now retries on 429/5xx with exponential backoff (up to 5 retries) instead of sleeping once and raising. Watchlist scan delays scale dynamically based on lookback setting and artist count to prevent sustained API pressure. A circuit breaker pauses the scan after consecutive rate-limit failures.
This commit is contained in:
Broque Thomas 2026-02-17 16:45:09 -08:00
parent 0f18b12967
commit 308f0f9711
2 changed files with 155 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

73
core/spotify_client.py
View file

@ -21,34 +21,57 @@ _request_queue = queue.Queue()
_queue_processor_running = False
def rate_limited(func):
"""Decorator to enforce rate limiting on Spotify API calls"""
"""Decorator to enforce rate limiting on Spotify API calls with retry and exponential backoff"""
@wraps(func)
def wrapper(*args, **kwargs):
global _last_api_call_time
with _api_call_lock:
current_time = time.time()
time_since_last_call = current_time - _last_api_call_time
if time_since_last_call < MIN_API_INTERVAL:
sleep_time = MIN_API_INTERVAL - time_since_last_call
time.sleep(sleep_time)
_last_api_call_time = time.time()
try:
result = func(*args, **kwargs)
return result
except Exception as e:
# Implement exponential backoff for API errors
if "rate limit" in str(e).lower() or "429" in str(e):
logger.warning(f"Rate limit hit, implementing backoff: {e}")
# Use longer backoff to avoid getting banned
time.sleep(3.0) # Wait 3 seconds before retrying
elif "503" in str(e) or "502" in str(e):
logger.warning(f"Spotify service error, backing off: {e}")
time.sleep(2.0) # Wait 2 seconds for service errors
raise e
max_retries = 5
for attempt in range(max_retries + 1):
# Enforce minimum interval between API calls
with _api_call_lock:
current_time = time.time()
time_since_last_call = current_time - _last_api_call_time
if time_since_last_call < MIN_API_INTERVAL:
sleep_time = MIN_API_INTERVAL - time_since_last_call
time.sleep(sleep_time)
_last_api_call_time = time.time()
try:
return func(*args, **kwargs)
except Exception as e:
error_str = str(e).lower()
is_rate_limit = "rate limit" in error_str or "429" in str(e)
is_server_error = "502" in str(e) or "503" in str(e)
if is_rate_limit and attempt < max_retries:
# Try to extract Retry-After from spotipy exception headers
retry_after = None
if hasattr(e, 'headers') and e.headers:
retry_after = e.headers.get('Retry-After') or e.headers.get('retry-after')
if retry_after:
try:
delay = int(retry_after) + 1
except (ValueError, TypeError):
delay = 3.0 * (2 ** attempt)
else:
delay = 3.0 * (2 ** attempt) # 3, 6, 12, 24, 48
logger.warning(f"Spotify rate limit hit, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries}): {func.__name__}")
time.sleep(delay)
continue
elif is_server_error and attempt < max_retries:
delay = 2.0 * (2 ** attempt) # 2, 4, 8, 16, 32
logger.warning(f"Spotify server error, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries}): {func.__name__}")
time.sleep(delay)
continue
raise
return wrapper
@dataclass

118
web_server.py
View file

@ -19239,7 +19239,37 @@ def start_watchlist_scan():
})
scan_results = []
# Dynamic delay calculation based on scan scope
lookback_period = scanner._get_lookback_period_setting()
is_full_discography = (lookback_period == 'all')
artist_count = len(watchlist_artists)
base_artist_delay = 2.0
base_album_delay = 0.5
# Scale up for full discography (way more albums per artist)
if is_full_discography:
base_artist_delay *= 2.0
base_album_delay *= 2.0
# Scale up further for large artist counts (sustained API pressure)
if artist_count > 200:
base_artist_delay *= 1.5
base_album_delay *= 1.25
elif artist_count > 100:
base_artist_delay *= 1.25
artist_delay = base_artist_delay
album_delay = base_album_delay
print(f"📊 Scan parameters: {artist_count} artists, lookback={lookback_period}, "
f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album")
# Circuit breaker: pause scan on consecutive rate-limit failures
consecutive_failures = 0
CIRCUIT_BREAKER_THRESHOLD = 3
circuit_breaker_pause = 60 # seconds, doubles each trigger, max 600s
for i, artist in enumerate(watchlist_artists):
try:
# Fetch artist image using provider-aware method
@ -19335,14 +19365,14 @@ def start_watchlist_scan():
if len(watchlist_scan_state['recent_wishlist_additions']) > 10:
watchlist_scan_state['recent_wishlist_additions'].pop()
# Small delay between albums
# Rate-limited delay between albums
import time
time.sleep(0.5)
time.sleep(album_delay)
except Exception as e:
print(f"Error checking album {album.name}: {e}")
continue
# Update scan timestamp
scanner.update_artist_scan_timestamp(artist)
@ -19381,10 +19411,28 @@ def start_watchlist_scan():
# Delay between artists
if i < len(watchlist_artists) - 1:
watchlist_scan_state['current_phase'] = 'rate_limiting'
time.sleep(2.0)
time.sleep(artist_delay)
# Reset circuit breaker on successful artist scan
consecutive_failures = 0
circuit_breaker_pause = 60
except Exception as e:
print(f"Error scanning artist {artist.artist_name}: {e}")
# Circuit breaker: detect consecutive rate-limit failures
error_str = str(e).lower()
if "429" in error_str or "rate limit" in error_str:
consecutive_failures += 1
if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD:
print(f"🛑 Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s")
watchlist_scan_state['current_phase'] = 'circuit_breaker_pause'
time.sleep(circuit_breaker_pause)
circuit_breaker_pause = min(circuit_breaker_pause * 2, 600)
consecutive_failures = 0
else:
consecutive_failures = 0
scan_results.append(type('ScanResult', (), {
'artist_name': artist.artist_name,
'spotify_artist_id': artist.spotify_artist_id,
@ -19394,7 +19442,7 @@ def start_watchlist_scan():
'success': False,
'error_message': str(e)
})())
# Store final results
watchlist_scan_state['status'] = 'completed'
watchlist_scan_state['results'] = scan_results
@ -19957,6 +20005,36 @@ def _process_watchlist_scan_automatically():
scan_results = []
# Dynamic delay calculation based on scan scope
lookback_period = scanner._get_lookback_period_setting()
is_full_discography = (lookback_period == 'all')
artist_count = len(watchlist_artists)
base_artist_delay = 2.0
base_album_delay = 0.5
# Scale up for full discography (way more albums per artist)
if is_full_discography:
base_artist_delay *= 2.0
base_album_delay *= 2.0
# Scale up further for large artist counts (sustained API pressure)
if artist_count > 200:
base_artist_delay *= 1.5
base_album_delay *= 1.25
elif artist_count > 100:
base_artist_delay *= 1.25
artist_delay = base_artist_delay
album_delay = base_album_delay
print(f"📊 [Auto-Watchlist] Scan parameters: {artist_count} artists, lookback={lookback_period}, "
f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album")
# Circuit breaker: pause scan on consecutive rate-limit failures
consecutive_failures = 0
CIRCUIT_BREAKER_THRESHOLD = 3
circuit_breaker_pause = 60 # seconds, doubles each trigger, max 600s
# Scan each artist with detailed tracking
for i, artist in enumerate(watchlist_artists):
try:
@ -20053,9 +20131,9 @@ def _process_watchlist_scan_automatically():
if len(watchlist_scan_state['recent_wishlist_additions']) > 10:
watchlist_scan_state['recent_wishlist_additions'].pop()
# Small delay between albums
# Rate-limited delay between albums
import time
time.sleep(0.5)
time.sleep(album_delay)
except Exception as e:
print(f"Error checking album {album.name}: {e}")
@ -20080,10 +20158,28 @@ def _process_watchlist_scan_automatically():
# Delay between artists
if i < len(watchlist_artists) - 1:
watchlist_scan_state['current_phase'] = 'rate_limiting'
time.sleep(2.0)
time.sleep(artist_delay)
# Reset circuit breaker on successful artist scan
consecutive_failures = 0
circuit_breaker_pause = 60
except Exception as e:
print(f"Error scanning artist {artist.artist_name}: {e}")
# Circuit breaker: detect consecutive rate-limit failures
error_str = str(e).lower()
if "429" in error_str or "rate limit" in error_str:
consecutive_failures += 1
if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD:
print(f"🛑 [Auto-Watchlist] Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s")
watchlist_scan_state['current_phase'] = 'circuit_breaker_pause'
time.sleep(circuit_breaker_pause)
circuit_breaker_pause = min(circuit_breaker_pause * 2, 600)
consecutive_failures = 0
else:
consecutive_failures = 0
scan_results.append(type('ScanResult', (), {
'artist_name': artist.artist_name,
'spotify_artist_id': artist.spotify_artist_id,