Add Spotify rate limit modal with live countdown and ban duration escalation

core/spotify_client.py

@@ -28,6 +28,8 @@ _rate_limit_retry_after = 0  # Original Retry-After value in seconds
 _rate_limit_endpoint = None  # Which function triggered the ban
 _rate_limit_set_at = 0       # When the ban was set
 _rate_limit_ban_ended_at = 0  # When the last ban expired naturally (for post-ban cooldown)
+_rate_limit_hit_count = 0    # How many times we've been rate limited recently (for escalation)
+_rate_limit_first_hit = 0    # Timestamp of the first hit in the current escalation window
 
 # Threshold: if Retry-After exceeds this, activate global ban instead of sleeping
 _LONG_RATE_LIMIT_THRESHOLD = 60  # seconds
@@ -37,6 +39,12 @@ _LONG_RATE_LIMIT_THRESHOLD = 60  # seconds
 # cooldown outlasts the Retry-After value they sent us.
 _POST_BAN_COOLDOWN = 300  # 5 minutes
 
+# Escalation: if we get rate limited again within this window, increase ban duration
+_ESCALATION_WINDOW = 3600   # 1 hour — if re-limited within this, escalate
+_ESCALATION_MAX = 14400     # 4 hours max ban
+_BASE_UNKNOWN_BAN = 1800    # 30 min default when Retry-After header is missing
+_BASE_MAX_RETRIES_BAN = 3600  # 1 hour default when spotipy exhausted all retries
+
 
 class SpotifyRateLimitError(Exception):
     """Raised when Spotify API calls are blocked due to active global rate limit ban."""
@@ -46,11 +54,32 @@ class SpotifyRateLimitError(Exception):
         super().__init__(f"Spotify rate limited for {retry_after}s (triggered by {endpoint})")
 
 
-def _set_global_rate_limit(retry_after_seconds, endpoint_name):
+def _set_global_rate_limit(retry_after_seconds, endpoint_name, has_real_header=False):
-    """Activate the global rate limit ban."""
+    """Activate the global rate limit ban. Escalates duration on repeated hits."""
     global _rate_limit_until, _rate_limit_retry_after, _rate_limit_endpoint, _rate_limit_set_at
+    global _rate_limit_hit_count, _rate_limit_first_hit
     with _rate_limit_lock:
         now = time.time()
+
+        # Escalation: if we're hitting rate limits repeatedly, increase the ban
+        if not has_real_header:
+            # Only escalate when we don't have a real Retry-After (i.e., we're guessing)
+            if now - _rate_limit_first_hit < _ESCALATION_WINDOW and _rate_limit_first_hit > 0:
+                _rate_limit_hit_count += 1
+            else:
+                # New escalation window
+                _rate_limit_hit_count = 1
+                _rate_limit_first_hit = now
+
+            if _rate_limit_hit_count > 1:
+                # Double the ban for each repeated hit, up to max
+                escalated = retry_after_seconds * (2 ** (_rate_limit_hit_count - 1))
+                retry_after_seconds = min(escalated, _ESCALATION_MAX)
+                logger.warning(
+                    f"Rate limit escalation: hit #{_rate_limit_hit_count} within window, "
+                    f"ban escalated to {retry_after_seconds}s"
+                )
+
         new_until = now + retry_after_seconds
         # Only update if this extends the existing ban
         if new_until > _rate_limit_until:
@@ -125,12 +154,15 @@ def _clear_rate_limit():
     """Manually clear the global rate limit ban AND post-ban cooldown.
     Used by disconnect/reconnect so the user can immediately retry."""
     global _rate_limit_until, _rate_limit_retry_after, _rate_limit_endpoint, _rate_limit_set_at, _rate_limit_ban_ended_at
+    global _rate_limit_hit_count, _rate_limit_first_hit
     with _rate_limit_lock:
         _rate_limit_until = 0
         _rate_limit_retry_after = 0
         _rate_limit_endpoint = None
         _rate_limit_set_at = 0
         _rate_limit_ban_ended_at = 0
+        _rate_limit_hit_count = 0
+        _rate_limit_first_hit = 0
     logger.info("Global rate limit ban cleared (including post-ban cooldown)")
 
 
@@ -138,23 +170,34 @@ def _detect_and_set_rate_limit(exception, endpoint_name="unknown"):
     """Check if a Spotify exception is a 429 rate limit and activate global ban if so.
     Returns True if rate limit was detected."""
     error_str = str(exception)
-    if "429" in error_str or "rate limit" in error_str.lower():
+    # Check both string matching and http_status attribute (SpotifyException has it)
-        # Try to extract Retry-After
+    is_429 = getattr(exception, 'http_status', None) == 429
+    is_rate_limit_str = "429" in error_str or "rate limit" in error_str.lower()
+
+    if is_429 or is_rate_limit_str:
+        # Try to extract Retry-After from exception headers
         retry_after = None
+        has_real_header = False
         if hasattr(exception, 'headers') and exception.headers:
             retry_after = exception.headers.get('Retry-After') or exception.headers.get('retry-after')
+
         if retry_after:
             try:
                 delay = int(retry_after)
+                has_real_header = True
+                logger.info(f"Rate limit detected on {endpoint_name} — Retry-After header: {delay}s")
             except (ValueError, TypeError):
-                delay = 300  # Default 5 min if can't parse
+                delay = _BASE_UNKNOWN_BAN
+                logger.warning(f"Rate limit detected on {endpoint_name} — unparseable Retry-After: {retry_after}")
         else:
-            # Spotipy "Max Retries" means it already exhausted retries — assume long ban
+            # No Retry-After header available
             if "max retries" in error_str.lower():
-                delay = 600  # 10 min default for exhausted retries
+                delay = _BASE_MAX_RETRIES_BAN  # 1 hour — retries exhausted
             else:
-                delay = 300
+                delay = _BASE_UNKNOWN_BAN  # 30 min
-        _set_global_rate_limit(delay, endpoint_name)
+            logger.warning(f"Rate limit detected on {endpoint_name} — no Retry-After header, using {delay}s default")
+
+        _set_global_rate_limit(delay, endpoint_name, has_real_header=has_real_header)
         return True
     return False
 
@@ -212,7 +255,7 @@ def rate_limited(func):
 
                         # If Retry-After is long, activate global ban instead of sleeping
                         if delay and delay > _LONG_RATE_LIMIT_THRESHOLD:
-                            _set_global_rate_limit(delay, func.__name__)
+                            _set_global_rate_limit(delay, func.__name__, has_real_header=True)
                             raise SpotifyRateLimitError(delay, func.__name__)
 
                         if delay:
@@ -226,6 +269,11 @@ def rate_limited(func):
                         logger.warning(f"Spotify rate limit hit, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries}): {func.__name__}")
                         time.sleep(delay)
                         continue
+                    else:
+                        # All retries exhausted on 429s — activate global ban.
+                        # Don't trust the Retry-After header here — we already retried
+                        # with it multiple times and still got 429'd, so it's too short.
+                        _set_global_rate_limit(_BASE_MAX_RETRIES_BAN, func.__name__)
 
                 elif is_server_error and attempt < max_retries:
                     delay = 2.0 * (2 ** attempt)  # 2, 4, 8, 16, 32
@@ -481,13 +529,16 @@ class SpotifyClient:
                 retry_after = None
                 if hasattr(e, 'headers') and e.headers:
                     retry_after = e.headers.get('Retry-After') or e.headers.get('retry-after')
+                has_real_header = False
                 try:
                     delay = int(retry_after) if retry_after else 0
+                    if retry_after:
+                        has_real_header = True
                 except (ValueError, TypeError):
                     delay = 0
-                # Minimum 10 minutes for auth probe 429s — these indicate persistent throttling
+                # Minimum 30 min for auth probe 429s — these indicate persistent throttling
-                ban_duration = max(delay, 600)
+                ban_duration = max(delay, _BASE_UNKNOWN_BAN)
-                _set_global_rate_limit(ban_duration, 'is_spotify_authenticated')
+                _set_global_rate_limit(ban_duration, 'is_spotify_authenticated', has_real_header=has_real_header)
                 logger.warning(f"Auth probe rate limited — activating {ban_duration}s global ban")
                 result = True
             else:

webui/index.html

@@ -5248,6 +5248,48 @@
         </div>
     </div>
 
+    <!-- Spotify Rate Limit Modal -->
+    <div class="modal-overlay hidden" id="rate-limit-modal-overlay">
+        <div class="confirm-modal rate-limit-modal">
+            <div class="confirm-modal-header rate-limit-modal-header">
+                <div class="rate-limit-title-row">
+                    <svg class="rate-limit-icon" viewBox="0 0 24 24" width="22" height="22" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+                        <path d="M10.29 3.86L1.82 18a2 2 0 0 0 1.71 3h16.94a2 2 0 0 0 1.71-3L13.71 3.86a2 2 0 0 0-3.42 0z"/>
+                        <line x1="12" y1="9" x2="12" y2="13"/>
+                        <line x1="12" y1="17" x2="12.01" y2="17"/>
+                    </svg>
+                    <h2>Spotify Rate Limited</h2>
+                </div>
+                <button class="confirm-modal-close" onclick="closeRateLimitModal()">✕</button>
+            </div>
+            <div class="confirm-modal-content rate-limit-modal-content">
+                <p class="rate-limit-description">Spotify has temporarily blocked API access. All Spotify features (search, enrichment, playlists) are paused until the ban expires.</p>
+                <div class="rate-limit-details">
+                    <div class="rate-limit-detail-row">
+                        <span class="rate-limit-label">Ban Duration</span>
+                        <span class="rate-limit-value" id="rate-limit-ban-duration">—</span>
+                    </div>
+                    <div class="rate-limit-detail-row">
+                        <span class="rate-limit-label">Triggered By</span>
+                        <span class="rate-limit-value rate-limit-endpoint" id="rate-limit-endpoint">—</span>
+                    </div>
+                    <div class="rate-limit-detail-row">
+                        <span class="rate-limit-label">Time Remaining</span>
+                        <span class="rate-limit-value rate-limit-countdown" id="rate-limit-countdown">—</span>
+                    </div>
+                </div>
+                <p class="rate-limit-hint">You can wait for the ban to expire (the app uses Apple Music in the meantime) or disconnect Spotify to clear the ban immediately.</p>
+            </div>
+            <div class="confirm-modal-actions rate-limit-modal-actions">
+                <button class="modal-button modal-button--secondary" onclick="closeRateLimitModal()">Dismiss</button>
+                <button class="modal-button rate-limit-disconnect-btn" onclick="disconnectSpotifyFromRateLimit()">
+                    Disconnect Spotify
+                    <span class="rate-limit-disconnect-sub">Clear ban, pause enrichment &amp; switch to iTunes</span>
+                </button>
+            </div>
+        </div>
+    </div>
+
     <script src="{{ url_for('static', filename='vendor/socket.io.min.js') }}"></script>
     <script src="{{ url_for('static', filename='script.js') }}"></script>
     <script src="{{ url_for('static', filename='docs.js') }}"></script>

webui/static/script.js

@@ -295,12 +295,16 @@ function handleServiceStatusUpdate(data) {
         if (_spotifyRateLimitShown && !_spotifyInCooldown) {
             _spotifyRateLimitShown = false;
             _spotifyInCooldown = true;
+            closeRateLimitModal();
             showToast('Spotify ban expired \u2014 recovering shortly', 'info');
         }
     } else {
         if (_spotifyInCooldown) {
             _spotifyInCooldown = false;
             showToast('Spotify access restored', 'success');
+            if (currentPage === 'discover') {
+                loadDiscoverPage();
+            }
         } else if (_spotifyRateLimitShown) {
             handleSpotifyRateLimit(null);
         }
@@ -5799,20 +5803,102 @@ async function disconnectSpotify() {
 // ── Spotify Rate Limit Handling ───────────────────────────────────────────
 let _spotifyRateLimitShown = false;
 let _spotifyInCooldown = false;
+let _rateLimitModalOpen = false;
+let _rateLimitCountdownInterval = null;
+let _rateLimitExpiresAt = 0;
 
 function handleSpotifyRateLimit(rateLimitInfo) {
     if (!rateLimitInfo || !rateLimitInfo.active) {
         if (_spotifyRateLimitShown) {
             _spotifyRateLimitShown = false;
+            closeRateLimitModal();
             showToast('Spotify access restored', 'success');
+            // Refresh discover page if user is on it — data source switched back to Spotify
+            if (currentPage === 'discover') {
+                console.log('Spotify restored — refreshing discover page data');
+                loadDiscoverPage();
+            }
         }
         return;
     }
+    // Update countdown if modal is open (status pushes every 10s keep it accurate)
+    if (_rateLimitModalOpen && rateLimitInfo.remaining_seconds) {
+        _rateLimitExpiresAt = Date.now() + (rateLimitInfo.remaining_seconds * 1000);
+    }
     if (!_spotifyRateLimitShown) {
         _spotifyRateLimitShown = true;
         _spotifyInCooldown = false;
-        const duration = formatRateLimitDuration(rateLimitInfo.remaining_seconds);
+        showRateLimitModal(rateLimitInfo);
-        showToast(`Spotify rate limited (${duration}) \u2014 using Apple Music`, 'warning');
+        // Refresh discover page if user is on it — data source switched to iTunes
+        if (currentPage === 'discover') {
+            console.log('Spotify rate limited — refreshing discover page with iTunes data');
+            loadDiscoverPage();
+        }
+    }
+}
+
+function showRateLimitModal(rateLimitInfo) {
+    const overlay = document.getElementById('rate-limit-modal-overlay');
+    if (!overlay) return;
+
+    // Populate details
+    const banDuration = document.getElementById('rate-limit-ban-duration');
+    const endpoint = document.getElementById('rate-limit-endpoint');
+    const countdown = document.getElementById('rate-limit-countdown');
+
+    banDuration.textContent = formatRateLimitDuration(rateLimitInfo.retry_after || rateLimitInfo.remaining_seconds);
+    endpoint.textContent = rateLimitInfo.endpoint || 'unknown';
+    countdown.textContent = formatRateLimitDuration(rateLimitInfo.remaining_seconds);
+
+    // Set expiry for live countdown
+    _rateLimitExpiresAt = Date.now() + (rateLimitInfo.remaining_seconds * 1000);
+
+    // Start live countdown timer
+    if (_rateLimitCountdownInterval) clearInterval(_rateLimitCountdownInterval);
+    _rateLimitCountdownInterval = setInterval(() => {
+        const remaining = Math.max(0, Math.round((_rateLimitExpiresAt - Date.now()) / 1000));
+        countdown.textContent = formatRateLimitDuration(remaining);
+        if (remaining <= 0) {
+            clearInterval(_rateLimitCountdownInterval);
+            _rateLimitCountdownInterval = null;
+        }
+    }, 1000);
+
+    overlay.classList.remove('hidden');
+    _rateLimitModalOpen = true;
+}
+
+function closeRateLimitModal() {
+    const overlay = document.getElementById('rate-limit-modal-overlay');
+    if (overlay) overlay.classList.add('hidden');
+    if (_rateLimitCountdownInterval) {
+        clearInterval(_rateLimitCountdownInterval);
+        _rateLimitCountdownInterval = null;
+    }
+    _rateLimitModalOpen = false;
+}
+
+async function disconnectSpotifyFromRateLimit() {
+    closeRateLimitModal();
+    try {
+        showLoadingOverlay('Disconnecting Spotify...');
+        const response = await fetch('/api/spotify/disconnect', { method: 'POST' });
+        const data = await response.json();
+        if (data.success) {
+            _spotifyRateLimitShown = false;
+            showToast('Spotify disconnected. Now using Apple Music/iTunes.', 'success');
+            await fetchAndUpdateServiceStatus();
+            if (currentPage === 'discover') {
+                loadDiscoverPage();
+            }
+        } else {
+            showToast(`Failed to disconnect: ${data.error}`, 'error');
+        }
+    } catch (error) {
+        console.error('Error disconnecting Spotify:', error);
+        showToast('Failed to disconnect Spotify', 'error');
+    } finally {
+        hideLoadingOverlay();
     }
 }
 

webui/static/style.css

@@ -4318,6 +4318,139 @@ body {
     border-bottom-right-radius: 20px;
 }
 
+/* ── Spotify Rate Limit Modal ─────────────────────────────── */
+
+.rate-limit-modal {
+    width: 500px;
+    border-color: rgba(255, 170, 0, 0.2) !important;
+    box-shadow:
+        0 25px 80px rgba(0, 0, 0, 0.7),
+        0 0 0 1px rgba(255, 170, 0, 0.1),
+        0 0 40px rgba(255, 170, 0, 0.04) !important;
+}
+
+.rate-limit-modal-header {
+    border-bottom-color: rgba(255, 170, 0, 0.15) !important;
+    background: linear-gradient(135deg, rgba(255, 170, 0, 0.07) 0%, transparent 60%) !important;
+}
+
+.rate-limit-title-row {
+    display: flex;
+    align-items: center;
+    gap: 10px;
+}
+
+.rate-limit-icon {
+    color: #ffaa00;
+    flex-shrink: 0;
+    filter: drop-shadow(0 0 6px rgba(255, 170, 0, 0.3));
+}
+
+.rate-limit-title-row h2 {
+    color: #ffaa00;
+    font-size: 18px;
+    font-weight: 700;
+    margin: 0;
+    letter-spacing: -0.3px;
+}
+
+.rate-limit-description {
+    color: rgba(255, 255, 255, 0.7);
+    font-size: 13.5px;
+    line-height: 1.6;
+    margin: 0 0 20px 0;
+}
+
+.rate-limit-details {
+    background: rgba(255, 170, 0, 0.03);
+    border: 1px solid rgba(255, 170, 0, 0.1);
+    border-radius: 12px;
+    padding: 4px 18px;
+    margin-bottom: 18px;
+}
+
+.rate-limit-detail-row {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 10px 0;
+}
+
+.rate-limit-detail-row:not(:last-child) {
+    border-bottom: 1px solid rgba(255, 255, 255, 0.05);
+}
+
+.rate-limit-label {
+    color: rgba(255, 255, 255, 0.45);
+    font-size: 13px;
+    font-weight: 500;
+}
+
+.rate-limit-value {
+    color: rgba(255, 255, 255, 0.9);
+    font-size: 13.5px;
+    font-weight: 600;
+}
+
+.rate-limit-endpoint {
+    font-family: 'SF Mono', 'Fira Code', 'Consolas', monospace;
+    font-size: 12px;
+    color: rgba(255, 170, 0, 0.85);
+    background: rgba(255, 170, 0, 0.1);
+    padding: 3px 10px;
+    border-radius: 6px;
+    border: 1px solid rgba(255, 170, 0, 0.12);
+}
+
+.rate-limit-countdown {
+    color: #ffaa00;
+    font-size: 14px;
+    font-weight: 700;
+    font-variant-numeric: tabular-nums;
+    letter-spacing: 0.3px;
+}
+
+.rate-limit-hint {
+    color: rgba(255, 255, 255, 0.4);
+    font-size: 12.5px;
+    line-height: 1.55;
+    margin: 0;
+}
+
+.rate-limit-modal-actions {
+    border-top-color: rgba(255, 170, 0, 0.1) !important;
+    justify-content: space-between !important;
+}
+
+.rate-limit-disconnect-btn {
+    background: rgba(255, 170, 0, 0.1) !important;
+    color: #ffaa00 !important;
+    border: 1px solid rgba(255, 170, 0, 0.25) !important;
+    border-radius: 10px !important;
+    padding: 10px 20px !important;
+    font-size: 13.5px !important;
+    font-weight: 600 !important;
+    cursor: pointer;
+    transition: all 0.2s ease;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    gap: 3px;
+}
+
+.rate-limit-disconnect-btn:hover {
+    background: rgba(255, 170, 0, 0.18) !important;
+    border-color: rgba(255, 170, 0, 0.4) !important;
+    box-shadow: 0 0 20px rgba(255, 170, 0, 0.08);
+}
+
+.rate-limit-disconnect-sub {
+    font-size: 10.5px;
+    font-weight: 400;
+    color: rgba(255, 170, 0, 0.55);
+    letter-spacing: 0.1px;
+}
+
 /* GUI-Matching Search Results Styling */
 
 /* Single Track Card (SearchResultItem) */