diff --git a/core/spotify_client.py b/core/spotify_client.py index 815409aa..9b3e4eb1 100644 --- a/core/spotify_client.py +++ b/core/spotify_client.py @@ -28,6 +28,8 @@ _rate_limit_retry_after = 0 # Original Retry-After value in seconds _rate_limit_endpoint = None # Which function triggered the ban _rate_limit_set_at = 0 # When the ban was set _rate_limit_ban_ended_at = 0 # When the last ban expired naturally (for post-ban cooldown) +_rate_limit_hit_count = 0 # How many times we've been rate limited recently (for escalation) +_rate_limit_first_hit = 0 # Timestamp of the first hit in the current escalation window # Threshold: if Retry-After exceeds this, activate global ban instead of sleeping _LONG_RATE_LIMIT_THRESHOLD = 60 # seconds @@ -37,6 +39,12 @@ _LONG_RATE_LIMIT_THRESHOLD = 60 # seconds # cooldown outlasts the Retry-After value they sent us. _POST_BAN_COOLDOWN = 300 # 5 minutes +# Escalation: if we get rate limited again within this window, increase ban duration +_ESCALATION_WINDOW = 3600 # 1 hour — if re-limited within this, escalate +_ESCALATION_MAX = 14400 # 4 hours max ban +_BASE_UNKNOWN_BAN = 1800 # 30 min default when Retry-After header is missing +_BASE_MAX_RETRIES_BAN = 3600 # 1 hour default when spotipy exhausted all retries + class SpotifyRateLimitError(Exception): """Raised when Spotify API calls are blocked due to active global rate limit ban.""" @@ -46,11 +54,32 @@ class SpotifyRateLimitError(Exception): super().__init__(f"Spotify rate limited for {retry_after}s (triggered by {endpoint})") -def _set_global_rate_limit(retry_after_seconds, endpoint_name): - """Activate the global rate limit ban.""" +def _set_global_rate_limit(retry_after_seconds, endpoint_name, has_real_header=False): + """Activate the global rate limit ban. Escalates duration on repeated hits.""" global _rate_limit_until, _rate_limit_retry_after, _rate_limit_endpoint, _rate_limit_set_at + global _rate_limit_hit_count, _rate_limit_first_hit with _rate_limit_lock: now = time.time() + + # Escalation: if we're hitting rate limits repeatedly, increase the ban + if not has_real_header: + # Only escalate when we don't have a real Retry-After (i.e., we're guessing) + if now - _rate_limit_first_hit < _ESCALATION_WINDOW and _rate_limit_first_hit > 0: + _rate_limit_hit_count += 1 + else: + # New escalation window + _rate_limit_hit_count = 1 + _rate_limit_first_hit = now + + if _rate_limit_hit_count > 1: + # Double the ban for each repeated hit, up to max + escalated = retry_after_seconds * (2 ** (_rate_limit_hit_count - 1)) + retry_after_seconds = min(escalated, _ESCALATION_MAX) + logger.warning( + f"Rate limit escalation: hit #{_rate_limit_hit_count} within window, " + f"ban escalated to {retry_after_seconds}s" + ) + new_until = now + retry_after_seconds # Only update if this extends the existing ban if new_until > _rate_limit_until: @@ -125,12 +154,15 @@ def _clear_rate_limit(): """Manually clear the global rate limit ban AND post-ban cooldown. Used by disconnect/reconnect so the user can immediately retry.""" global _rate_limit_until, _rate_limit_retry_after, _rate_limit_endpoint, _rate_limit_set_at, _rate_limit_ban_ended_at + global _rate_limit_hit_count, _rate_limit_first_hit with _rate_limit_lock: _rate_limit_until = 0 _rate_limit_retry_after = 0 _rate_limit_endpoint = None _rate_limit_set_at = 0 _rate_limit_ban_ended_at = 0 + _rate_limit_hit_count = 0 + _rate_limit_first_hit = 0 logger.info("Global rate limit ban cleared (including post-ban cooldown)") @@ -138,23 +170,34 @@ def _detect_and_set_rate_limit(exception, endpoint_name="unknown"): """Check if a Spotify exception is a 429 rate limit and activate global ban if so. Returns True if rate limit was detected.""" error_str = str(exception) - if "429" in error_str or "rate limit" in error_str.lower(): - # Try to extract Retry-After + # Check both string matching and http_status attribute (SpotifyException has it) + is_429 = getattr(exception, 'http_status', None) == 429 + is_rate_limit_str = "429" in error_str or "rate limit" in error_str.lower() + + if is_429 or is_rate_limit_str: + # Try to extract Retry-After from exception headers retry_after = None + has_real_header = False if hasattr(exception, 'headers') and exception.headers: retry_after = exception.headers.get('Retry-After') or exception.headers.get('retry-after') + if retry_after: try: delay = int(retry_after) + has_real_header = True + logger.info(f"Rate limit detected on {endpoint_name} — Retry-After header: {delay}s") except (ValueError, TypeError): - delay = 300 # Default 5 min if can't parse + delay = _BASE_UNKNOWN_BAN + logger.warning(f"Rate limit detected on {endpoint_name} — unparseable Retry-After: {retry_after}") else: - # Spotipy "Max Retries" means it already exhausted retries — assume long ban + # No Retry-After header available if "max retries" in error_str.lower(): - delay = 600 # 10 min default for exhausted retries + delay = _BASE_MAX_RETRIES_BAN # 1 hour — retries exhausted else: - delay = 300 - _set_global_rate_limit(delay, endpoint_name) + delay = _BASE_UNKNOWN_BAN # 30 min + logger.warning(f"Rate limit detected on {endpoint_name} — no Retry-After header, using {delay}s default") + + _set_global_rate_limit(delay, endpoint_name, has_real_header=has_real_header) return True return False @@ -212,7 +255,7 @@ def rate_limited(func): # If Retry-After is long, activate global ban instead of sleeping if delay and delay > _LONG_RATE_LIMIT_THRESHOLD: - _set_global_rate_limit(delay, func.__name__) + _set_global_rate_limit(delay, func.__name__, has_real_header=True) raise SpotifyRateLimitError(delay, func.__name__) if delay: @@ -226,6 +269,11 @@ def rate_limited(func): logger.warning(f"Spotify rate limit hit, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries}): {func.__name__}") time.sleep(delay) continue + else: + # All retries exhausted on 429s — activate global ban. + # Don't trust the Retry-After header here — we already retried + # with it multiple times and still got 429'd, so it's too short. + _set_global_rate_limit(_BASE_MAX_RETRIES_BAN, func.__name__) elif is_server_error and attempt < max_retries: delay = 2.0 * (2 ** attempt) # 2, 4, 8, 16, 32 @@ -481,13 +529,16 @@ class SpotifyClient: retry_after = None if hasattr(e, 'headers') and e.headers: retry_after = e.headers.get('Retry-After') or e.headers.get('retry-after') + has_real_header = False try: delay = int(retry_after) if retry_after else 0 + if retry_after: + has_real_header = True except (ValueError, TypeError): delay = 0 - # Minimum 10 minutes for auth probe 429s — these indicate persistent throttling - ban_duration = max(delay, 600) - _set_global_rate_limit(ban_duration, 'is_spotify_authenticated') + # Minimum 30 min for auth probe 429s — these indicate persistent throttling + ban_duration = max(delay, _BASE_UNKNOWN_BAN) + _set_global_rate_limit(ban_duration, 'is_spotify_authenticated', has_real_header=has_real_header) logger.warning(f"Auth probe rate limited — activating {ban_duration}s global ban") result = True else: diff --git a/webui/index.html b/webui/index.html index a51a6010..fb2c5573 100644 --- a/webui/index.html +++ b/webui/index.html @@ -5248,6 +5248,48 @@ + +
+ diff --git a/webui/static/script.js b/webui/static/script.js index 3e784674..39074dca 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -295,12 +295,16 @@ function handleServiceStatusUpdate(data) { if (_spotifyRateLimitShown && !_spotifyInCooldown) { _spotifyRateLimitShown = false; _spotifyInCooldown = true; + closeRateLimitModal(); showToast('Spotify ban expired \u2014 recovering shortly', 'info'); } } else { if (_spotifyInCooldown) { _spotifyInCooldown = false; showToast('Spotify access restored', 'success'); + if (currentPage === 'discover') { + loadDiscoverPage(); + } } else if (_spotifyRateLimitShown) { handleSpotifyRateLimit(null); } @@ -5799,20 +5803,102 @@ async function disconnectSpotify() { // ── Spotify Rate Limit Handling ─────────────────────────────────────────── let _spotifyRateLimitShown = false; let _spotifyInCooldown = false; +let _rateLimitModalOpen = false; +let _rateLimitCountdownInterval = null; +let _rateLimitExpiresAt = 0; function handleSpotifyRateLimit(rateLimitInfo) { if (!rateLimitInfo || !rateLimitInfo.active) { if (_spotifyRateLimitShown) { _spotifyRateLimitShown = false; + closeRateLimitModal(); showToast('Spotify access restored', 'success'); + // Refresh discover page if user is on it — data source switched back to Spotify + if (currentPage === 'discover') { + console.log('Spotify restored — refreshing discover page data'); + loadDiscoverPage(); + } } return; } + // Update countdown if modal is open (status pushes every 10s keep it accurate) + if (_rateLimitModalOpen && rateLimitInfo.remaining_seconds) { + _rateLimitExpiresAt = Date.now() + (rateLimitInfo.remaining_seconds * 1000); + } if (!_spotifyRateLimitShown) { _spotifyRateLimitShown = true; _spotifyInCooldown = false; - const duration = formatRateLimitDuration(rateLimitInfo.remaining_seconds); - showToast(`Spotify rate limited (${duration}) \u2014 using Apple Music`, 'warning'); + showRateLimitModal(rateLimitInfo); + // Refresh discover page if user is on it — data source switched to iTunes + if (currentPage === 'discover') { + console.log('Spotify rate limited — refreshing discover page with iTunes data'); + loadDiscoverPage(); + } + } +} + +function showRateLimitModal(rateLimitInfo) { + const overlay = document.getElementById('rate-limit-modal-overlay'); + if (!overlay) return; + + // Populate details + const banDuration = document.getElementById('rate-limit-ban-duration'); + const endpoint = document.getElementById('rate-limit-endpoint'); + const countdown = document.getElementById('rate-limit-countdown'); + + banDuration.textContent = formatRateLimitDuration(rateLimitInfo.retry_after || rateLimitInfo.remaining_seconds); + endpoint.textContent = rateLimitInfo.endpoint || 'unknown'; + countdown.textContent = formatRateLimitDuration(rateLimitInfo.remaining_seconds); + + // Set expiry for live countdown + _rateLimitExpiresAt = Date.now() + (rateLimitInfo.remaining_seconds * 1000); + + // Start live countdown timer + if (_rateLimitCountdownInterval) clearInterval(_rateLimitCountdownInterval); + _rateLimitCountdownInterval = setInterval(() => { + const remaining = Math.max(0, Math.round((_rateLimitExpiresAt - Date.now()) / 1000)); + countdown.textContent = formatRateLimitDuration(remaining); + if (remaining <= 0) { + clearInterval(_rateLimitCountdownInterval); + _rateLimitCountdownInterval = null; + } + }, 1000); + + overlay.classList.remove('hidden'); + _rateLimitModalOpen = true; +} + +function closeRateLimitModal() { + const overlay = document.getElementById('rate-limit-modal-overlay'); + if (overlay) overlay.classList.add('hidden'); + if (_rateLimitCountdownInterval) { + clearInterval(_rateLimitCountdownInterval); + _rateLimitCountdownInterval = null; + } + _rateLimitModalOpen = false; +} + +async function disconnectSpotifyFromRateLimit() { + closeRateLimitModal(); + try { + showLoadingOverlay('Disconnecting Spotify...'); + const response = await fetch('/api/spotify/disconnect', { method: 'POST' }); + const data = await response.json(); + if (data.success) { + _spotifyRateLimitShown = false; + showToast('Spotify disconnected. Now using Apple Music/iTunes.', 'success'); + await fetchAndUpdateServiceStatus(); + if (currentPage === 'discover') { + loadDiscoverPage(); + } + } else { + showToast(`Failed to disconnect: ${data.error}`, 'error'); + } + } catch (error) { + console.error('Error disconnecting Spotify:', error); + showToast('Failed to disconnect Spotify', 'error'); + } finally { + hideLoadingOverlay(); } } diff --git a/webui/static/style.css b/webui/static/style.css index d7ae0c46..acd53710 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -4318,6 +4318,139 @@ body { border-bottom-right-radius: 20px; } +/* ── Spotify Rate Limit Modal ─────────────────────────────── */ + +.rate-limit-modal { + width: 500px; + border-color: rgba(255, 170, 0, 0.2) !important; + box-shadow: + 0 25px 80px rgba(0, 0, 0, 0.7), + 0 0 0 1px rgba(255, 170, 0, 0.1), + 0 0 40px rgba(255, 170, 0, 0.04) !important; +} + +.rate-limit-modal-header { + border-bottom-color: rgba(255, 170, 0, 0.15) !important; + background: linear-gradient(135deg, rgba(255, 170, 0, 0.07) 0%, transparent 60%) !important; +} + +.rate-limit-title-row { + display: flex; + align-items: center; + gap: 10px; +} + +.rate-limit-icon { + color: #ffaa00; + flex-shrink: 0; + filter: drop-shadow(0 0 6px rgba(255, 170, 0, 0.3)); +} + +.rate-limit-title-row h2 { + color: #ffaa00; + font-size: 18px; + font-weight: 700; + margin: 0; + letter-spacing: -0.3px; +} + +.rate-limit-description { + color: rgba(255, 255, 255, 0.7); + font-size: 13.5px; + line-height: 1.6; + margin: 0 0 20px 0; +} + +.rate-limit-details { + background: rgba(255, 170, 0, 0.03); + border: 1px solid rgba(255, 170, 0, 0.1); + border-radius: 12px; + padding: 4px 18px; + margin-bottom: 18px; +} + +.rate-limit-detail-row { + display: flex; + justify-content: space-between; + align-items: center; + padding: 10px 0; +} + +.rate-limit-detail-row:not(:last-child) { + border-bottom: 1px solid rgba(255, 255, 255, 0.05); +} + +.rate-limit-label { + color: rgba(255, 255, 255, 0.45); + font-size: 13px; + font-weight: 500; +} + +.rate-limit-value { + color: rgba(255, 255, 255, 0.9); + font-size: 13.5px; + font-weight: 600; +} + +.rate-limit-endpoint { + font-family: 'SF Mono', 'Fira Code', 'Consolas', monospace; + font-size: 12px; + color: rgba(255, 170, 0, 0.85); + background: rgba(255, 170, 0, 0.1); + padding: 3px 10px; + border-radius: 6px; + border: 1px solid rgba(255, 170, 0, 0.12); +} + +.rate-limit-countdown { + color: #ffaa00; + font-size: 14px; + font-weight: 700; + font-variant-numeric: tabular-nums; + letter-spacing: 0.3px; +} + +.rate-limit-hint { + color: rgba(255, 255, 255, 0.4); + font-size: 12.5px; + line-height: 1.55; + margin: 0; +} + +.rate-limit-modal-actions { + border-top-color: rgba(255, 170, 0, 0.1) !important; + justify-content: space-between !important; +} + +.rate-limit-disconnect-btn { + background: rgba(255, 170, 0, 0.1) !important; + color: #ffaa00 !important; + border: 1px solid rgba(255, 170, 0, 0.25) !important; + border-radius: 10px !important; + padding: 10px 20px !important; + font-size: 13.5px !important; + font-weight: 600 !important; + cursor: pointer; + transition: all 0.2s ease; + display: flex; + flex-direction: column; + align-items: center; + gap: 3px; +} + +.rate-limit-disconnect-btn:hover { + background: rgba(255, 170, 0, 0.18) !important; + border-color: rgba(255, 170, 0, 0.4) !important; + box-shadow: 0 0 20px rgba(255, 170, 0, 0.08); +} + +.rate-limit-disconnect-sub { + font-size: 10.5px; + font-weight: 400; + color: rgba(255, 170, 0, 0.55); + letter-spacing: 0.1px; +} + /* GUI-Matching Search Results Styling */ /* Single Track Card (SearchResultItem) */