Make Spotify status updates event-driven

- move Spotify status publishing onto auth, disconnect, and rate-limit transitions
- keep dashboard and debug consumers on the shared cached snapshot
- leave only the initial snapshot seed as a fallback probe
This commit is contained in:
Antti Kettunen 2026-05-02 12:13:17 +03:00
parent cc13fb8f01
commit 36131656dd
No known key found for this signature in database
GPG key ID: C6B2A3D250359BD7
5 changed files with 253 additions and 68 deletions

View file

@ -300,23 +300,24 @@ def get_debug_info():
# API rate monitor — current calls/min, 24h totals, peaks, rate limit events # API rate monitor — current calls/min, 24h totals, peaks, rate limit events
try: try:
from core.api_call_tracker import api_call_tracker from core.api_call_tracker import api_call_tracker
from core.metadata.status import get_spotify_status
rates = api_call_tracker.get_all_rates() rates = api_call_tracker.get_all_rates()
info['api_rates'] = rates info['api_rates'] = rates
# Rich 24h debug summary with peaks, totals, per-endpoint breakdown, events # Rich 24h debug summary with peaks, totals, per-endpoint breakdown, events
info['api_debug_summary'] = api_call_tracker.get_debug_summary() info['api_debug_summary'] = api_call_tracker.get_debug_summary()
# Spotify rate limit details # Spotify rate limit details
if spotify_client: spotify_status = get_spotify_status(spotify_client=spotify_client)
rl_info = spotify_client.get_rate_limit_info() rl_info = spotify_status.get('rate_limit')
if rl_info: if spotify_status.get('rate_limited') and rl_info:
info['spotify_rate_limit'] = { info['spotify_rate_limit'] = {
'active': True, 'active': True,
'remaining_seconds': rl_info.get('remaining_seconds', 0), 'remaining_seconds': rl_info.get('remaining_seconds', 0),
'retry_after': rl_info.get('retry_after', 0), 'retry_after': rl_info.get('retry_after', 0),
'endpoint': rl_info.get('endpoint', ''), 'endpoint': rl_info.get('endpoint', ''),
'expires_at': rl_info.get('expires_at', ''), 'expires_at': rl_info.get('expires_at', ''),
} }
else: else:
info['spotify_rate_limit'] = {'active': False} info['spotify_rate_limit'] = {'active': False}
except Exception: except Exception:
info['api_rates'] = {} info['api_rates'] = {}
info['api_debug_summary'] = {} info['api_debug_summary'] = {}

View file

@ -42,8 +42,6 @@ from core.metadata.registry import (
) )
from core.metadata.status import ( from core.metadata.status import (
METADATA_SOURCE_STATUS_TTL, METADATA_SOURCE_STATUS_TTL,
SPOTIFY_STATUS_TTL_ACTIVE,
SPOTIFY_STATUS_TTL_IDLE,
get_metadata_source_status, get_metadata_source_status,
get_spotify_status, get_spotify_status,
get_status_snapshot, get_status_snapshot,
@ -98,7 +96,5 @@ __all__ = [
"register_profile_spotify_credentials_provider", "register_profile_spotify_credentials_provider",
"register_runtime_clients", "register_runtime_clients",
"resolve_album_reference", "resolve_album_reference",
"SPOTIFY_STATUS_TTL_ACTIVE",
"SPOTIFY_STATUS_TTL_IDLE",
"invalidate_metadata_status_caches", "invalidate_metadata_status_caches",
] ]

View file

@ -6,7 +6,6 @@ import threading
import time import time
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
from config.settings import config_manager
from utils.logging_config import get_logger from utils.logging_config import get_logger
from core.metadata.registry import get_primary_source_status from core.metadata.registry import get_primary_source_status
@ -14,8 +13,8 @@ from core.metadata.registry import get_primary_source_status
logger = get_logger("metadata.status") logger = get_logger("metadata.status")
METADATA_SOURCE_STATUS_TTL = 120 METADATA_SOURCE_STATUS_TTL = 120
SPOTIFY_STATUS_TTL_ACTIVE = 15
SPOTIFY_STATUS_TTL_IDLE = 300 _UNSET = object()
_status_lock = threading.RLock() _status_lock = threading.RLock()
_metadata_source_status_cache: Dict[str, Any] = { _metadata_source_status_cache: Dict[str, Any] = {
@ -32,21 +31,80 @@ _spotify_status_cache: Dict[str, Any] = {
"post_ban_cooldown": None, "post_ban_cooldown": None,
} }
_spotify_status_timestamp = 0.0 _spotify_status_timestamp = 0.0
_spotify_status_initialized = False
_spotify_rate_limit_expires_at = 0.0
def _get_config_value(key: str, default: Any = None) -> Any: _spotify_post_ban_cooldown_expires_at = 0.0
try:
return config_manager.get(key, default)
except Exception:
return default
def invalidate_metadata_status_caches() -> None: def invalidate_metadata_status_caches() -> None:
"""Mark the cached metadata-source and Spotify status snapshots stale.""" """Mark the cached metadata-source snapshot stale."""
global _metadata_source_status_timestamp, _spotify_status_timestamp global _metadata_source_status_timestamp
with _status_lock: with _status_lock:
_metadata_source_status_timestamp = 0.0 _metadata_source_status_timestamp = 0.0
_spotify_status_timestamp = 0.0
def publish_spotify_status(
*,
connected: Any = _UNSET,
authenticated: Any = _UNSET,
rate_limited: Any = _UNSET,
rate_limit: Any = _UNSET,
post_ban_cooldown: Any = _UNSET,
) -> Dict[str, Any]:
"""Update the cached Spotify status snapshot from an event."""
global _spotify_status_timestamp, _spotify_status_initialized
global _spotify_rate_limit_expires_at, _spotify_post_ban_cooldown_expires_at
with _status_lock:
if connected is not _UNSET:
_spotify_status_cache["connected"] = connected
if authenticated is not _UNSET:
_spotify_status_cache["authenticated"] = authenticated
if rate_limited is not _UNSET:
_spotify_status_cache["rate_limited"] = rate_limited
if rate_limit is not _UNSET:
_spotify_status_cache["rate_limit"] = rate_limit
if rate_limit and isinstance(rate_limit, dict):
_spotify_rate_limit_expires_at = float(rate_limit.get("expires_at") or 0.0)
else:
_spotify_rate_limit_expires_at = 0.0
if post_ban_cooldown is not _UNSET:
_spotify_status_cache["post_ban_cooldown"] = post_ban_cooldown
if post_ban_cooldown is not None:
_spotify_post_ban_cooldown_expires_at = time.time() + max(0, float(post_ban_cooldown))
else:
_spotify_post_ban_cooldown_expires_at = 0.0
_spotify_status_timestamp = time.time()
_spotify_status_initialized = True
_normalize_spotify_status_locked(_spotify_status_timestamp)
return dict(_spotify_status_cache)
def refresh_spotify_status_from_client(spotify_client: Optional[Any]) -> Dict[str, Any]:
"""Probe Spotify once to seed the cache when no event has populated it yet."""
if spotify_client is None:
with _status_lock:
return dict(_spotify_status_cache)
try:
is_rate_limited = spotify_client.is_rate_limited() if spotify_client else False
rate_limit_info = spotify_client.get_rate_limit_info() if (spotify_client and is_rate_limited) else None
cooldown_remaining = spotify_client.get_post_ban_cooldown_remaining() if spotify_client else 0
authenticated = spotify_client.is_spotify_authenticated() if spotify_client else False
except Exception as exc:
logger.debug("Spotify status probe failed: %s", exc)
authenticated = False
is_rate_limited = False
rate_limit_info = None
cooldown_remaining = 0
return publish_spotify_status(
connected=authenticated,
authenticated=authenticated,
rate_limited=is_rate_limited,
rate_limit=rate_limit_info,
post_ban_cooldown=cooldown_remaining if cooldown_remaining > 0 else None,
)
def get_metadata_source_status() -> Dict[str, Any]: def get_metadata_source_status() -> Dict[str, Any]:
@ -75,36 +133,42 @@ def get_metadata_source_status() -> Dict[str, Any]:
def get_spotify_status(spotify_client: Optional[Any] = None) -> Dict[str, Any]: def get_spotify_status(spotify_client: Optional[Any] = None) -> Dict[str, Any]:
"""Return a cached Spotify-specific status snapshot.""" """Return a cached Spotify-specific status snapshot."""
global _spotify_status_timestamp
current_time = time.time()
configured_source = _get_config_value("metadata.fallback_source", "deezer") or "deezer"
ttl = SPOTIFY_STATUS_TTL_ACTIVE if configured_source == "spotify" else SPOTIFY_STATUS_TTL_IDLE
with _status_lock: with _status_lock:
if _spotify_status_timestamp and current_time - _spotify_status_timestamp <= ttl: if _spotify_status_initialized:
_normalize_spotify_status_locked(time.time())
return dict(_spotify_status_cache) return dict(_spotify_status_cache)
try: return refresh_spotify_status_from_client(spotify_client)
is_rate_limited = spotify_client.is_rate_limited() if spotify_client else False
rate_limit_info = spotify_client.get_rate_limit_info() if (spotify_client and is_rate_limited) else None
cooldown_remaining = spotify_client.get_post_ban_cooldown_remaining() if spotify_client else 0
authenticated = spotify_client.is_spotify_authenticated() if spotify_client else False
with _status_lock:
_spotify_status_cache.update({
"connected": authenticated,
"authenticated": authenticated,
"rate_limited": is_rate_limited,
"rate_limit": rate_limit_info,
"post_ban_cooldown": cooldown_remaining if cooldown_remaining > 0 else None,
})
_spotify_status_timestamp = current_time
except Exception as exc:
logger.debug("Spotify status refresh failed: %s", exc)
with _status_lock: def _normalize_spotify_status_locked(current_time: float) -> None:
return dict(_spotify_status_cache) """Update derived Spotify status fields and clear expired ban state."""
global _spotify_rate_limit_expires_at, _spotify_post_ban_cooldown_expires_at
rate_limit = _spotify_status_cache.get("rate_limit")
if _spotify_status_cache.get("rate_limited") and rate_limit and isinstance(rate_limit, dict):
expires_at = float(rate_limit.get("expires_at") or 0.0)
if expires_at > 0:
remaining = int(max(0, expires_at - current_time))
if remaining > 0:
_spotify_status_cache["rate_limit"] = {**rate_limit, "remaining_seconds": remaining}
_spotify_rate_limit_expires_at = expires_at
else:
_spotify_status_cache["rate_limited"] = False
_spotify_status_cache["rate_limit"] = None
_spotify_rate_limit_expires_at = 0.0
elif _spotify_rate_limit_expires_at and current_time >= _spotify_rate_limit_expires_at:
_spotify_status_cache["rate_limited"] = False
_spotify_status_cache["rate_limit"] = None
_spotify_rate_limit_expires_at = 0.0
if _spotify_post_ban_cooldown_expires_at > 0:
remaining = int(max(0, _spotify_post_ban_cooldown_expires_at - current_time))
if remaining > 0:
_spotify_status_cache["post_ban_cooldown"] = remaining
else:
_spotify_status_cache["post_ban_cooldown"] = None
_spotify_post_ban_cooldown_expires_at = 0.0
def get_status_snapshot(spotify_client: Optional[Any] = None) -> Dict[str, Any]: def get_status_snapshot(spotify_client: Optional[Any] = None) -> Dict[str, Any]:

View file

@ -138,6 +138,18 @@ def _set_global_rate_limit(retry_after_seconds, endpoint_name, has_real_header=F
) )
except Exception: except Exception:
pass pass
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=True,
rate_limited=True,
rate_limit=_get_rate_limit_info(),
post_ban_cooldown=_get_post_ban_cooldown_remaining() or None,
)
except Exception:
pass
def _is_globally_rate_limited(): def _is_globally_rate_limited():
@ -210,6 +222,16 @@ def _clear_rate_limit():
_rate_limit_hit_count = 0 _rate_limit_hit_count = 0
_rate_limit_first_hit = 0 _rate_limit_first_hit = 0
logger.info("Global rate limit ban cleared (including post-ban cooldown)") logger.info("Global rate limit ban cleared (including post-ban cooldown)")
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
rate_limited=False,
rate_limit=None,
post_ban_cooldown=None,
)
except Exception:
pass
def _detect_and_set_rate_limit(exception, endpoint_name="unknown"): def _detect_and_set_rate_limit(exception, endpoint_name="unknown"):
@ -603,13 +625,38 @@ class SpotifyClient:
"""Check if Spotify client is specifically authenticated (not just iTunes fallback). """Check if Spotify client is specifically authenticated (not just iTunes fallback).
Results are cached for 60 seconds to avoid excessive API calls. Results are cached for 60 seconds to avoid excessive API calls.
During rate limit bans and post-ban cooldown, returns False without making API calls.""" During rate limit bans and post-ban cooldown, returns False without making API calls."""
rate_limited_state = False
if self.sp is None: if self.sp is None:
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=False,
rate_limited=_is_globally_rate_limited(),
rate_limit=_get_rate_limit_info(),
post_ban_cooldown=_get_post_ban_cooldown_remaining() or None,
)
except Exception:
pass
return False return False
# If globally rate limited, report as NOT authenticated so callers # If globally rate limited, report as NOT authenticated so callers
# skip Spotify and fall through to iTunes fallback naturally. # skip Spotify and fall through to iTunes fallback naturally.
# This prevents any API calls that could extend the ban. # This prevents any API calls that could extend the ban.
if _is_globally_rate_limited(): if _is_globally_rate_limited():
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=True,
rate_limited=True,
rate_limit=_get_rate_limit_info(),
post_ban_cooldown=_get_post_ban_cooldown_remaining() or None,
)
except Exception:
pass
return False return False
# Post-ban cooldown: after a ban expires, don't probe Spotify immediately. # Post-ban cooldown: after a ban expires, don't probe Spotify immediately.
@ -618,11 +665,35 @@ class SpotifyClient:
if _is_in_post_ban_cooldown(): if _is_in_post_ban_cooldown():
remaining = _get_post_ban_cooldown_remaining() remaining = _get_post_ban_cooldown_remaining()
logger.debug(f"Post-ban cooldown active ({remaining}s left), skipping auth probe") logger.debug(f"Post-ban cooldown active ({remaining}s left), skipping auth probe")
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=True,
rate_limited=False,
rate_limit=None,
post_ban_cooldown=remaining or None,
)
except Exception:
pass
return False return False
# Check cache first (lock only for brief read) # Check cache first (lock only for brief read)
with self._auth_cache_lock: with self._auth_cache_lock:
if self._auth_cached_result is not None and (time.time() - self._auth_cache_time) < self._AUTH_CACHE_TTL: if self._auth_cached_result is not None and (time.time() - self._auth_cache_time) < self._AUTH_CACHE_TTL:
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=self._auth_cached_result,
authenticated=self._auth_cached_result,
rate_limited=False,
rate_limit=None,
post_ban_cooldown=None,
)
except Exception:
pass
return self._auth_cached_result return self._auth_cached_result
# Cache miss — make API call outside the lock. # Cache miss — make API call outside the lock.
@ -637,6 +708,18 @@ class SpotifyClient:
with self._auth_cache_lock: with self._auth_cache_lock:
self._auth_cached_result = False self._auth_cached_result = False
self._auth_cache_time = time.time() self._auth_cache_time = time.time()
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=False,
rate_limited=False,
rate_limit=None,
post_ban_cooldown=None,
)
except Exception:
pass
return False return False
except Exception: except Exception:
pass pass
@ -668,6 +751,7 @@ class SpotifyClient:
ban_duration = max(delay, _BASE_UNKNOWN_BAN) ban_duration = max(delay, _BASE_UNKNOWN_BAN)
_set_global_rate_limit(ban_duration, 'is_spotify_authenticated', has_real_header=has_real_header) _set_global_rate_limit(ban_duration, 'is_spotify_authenticated', has_real_header=has_real_header)
logger.warning(f"Auth probe rate limited — activating {ban_duration}s global ban") logger.warning(f"Auth probe rate limited — activating {ban_duration}s global ban")
rate_limited_state = True
result = True result = True
else: else:
logger.debug(f"Spotify authentication check failed: {e}") logger.debug(f"Spotify authentication check failed: {e}")
@ -677,6 +761,19 @@ class SpotifyClient:
self._auth_cached_result = result self._auth_cached_result = result
self._auth_cache_time = time.time() self._auth_cache_time = time.time()
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=result,
authenticated=result,
rate_limited=rate_limited_state,
rate_limit=_get_rate_limit_info() if rate_limited_state else None,
post_ban_cooldown=None,
)
except Exception:
pass
return result return result
def disconnect(self): def disconnect(self):
@ -686,6 +783,18 @@ class SpotifyClient:
self.user_id = None self.user_id = None
self._invalidate_auth_cache() self._invalidate_auth_cache()
_clear_rate_limit() _clear_rate_limit()
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=False,
rate_limited=False,
rate_limit=None,
post_ban_cooldown=None,
)
except Exception:
pass
cache_path = 'config/.spotify_cache' cache_path = 'config/.spotify_cache'
try: try:

View file

@ -105,6 +105,8 @@ from core.metadata.registry import (
) )
from core.metadata.status import ( from core.metadata.status import (
get_status_snapshot as get_metadata_status_snapshot, get_status_snapshot as get_metadata_status_snapshot,
get_spotify_status,
publish_spotify_status,
invalidate_metadata_status_caches, invalidate_metadata_status_caches,
) )
from core.imports.context import ( from core.imports.context import (
@ -4152,6 +4154,14 @@ def handle_settings():
genius_worker._init_client() genius_worker._init_client()
if tidal_enrichment_worker: if tidal_enrichment_worker:
tidal_enrichment_worker.client = tidal_client tidal_enrichment_worker.client = tidal_client
if 'spotify' in new_settings:
publish_spotify_status(
connected=False,
authenticated=False,
rate_limited=False,
rate_limit=None,
post_ban_cooldown=None,
)
# Invalidate status cache so next poll reflects new settings (e.g. fallback source change) # Invalidate status cache so next poll reflects new settings (e.g. fallback source change)
invalidate_metadata_status_caches() invalidate_metadata_status_caches()
logger.info("Service clients re-initialized with new settings.") logger.info("Service clients re-initialized with new settings.")
@ -5906,15 +5916,20 @@ def spotify_disconnect():
def spotify_rate_limit_status(): def spotify_rate_limit_status():
"""Get Spotify rate limit ban details""" """Get Spotify rate limit ban details"""
try: try:
info = spotify_client.get_rate_limit_info() info = get_spotify_status(spotify_client=spotify_client)
rate_limit_info = info.get('rate_limit')
if info: if info:
return jsonify({ payload = {
'rate_limited': True, 'rate_limited': bool(info.get('rate_limited')),
'remaining_seconds': info['remaining_seconds'], }
'retry_after': info['retry_after'], if rate_limit_info:
'endpoint': info['endpoint'], payload.update({
'expires_at': info['expires_at'] 'remaining_seconds': rate_limit_info.get('remaining_seconds', 0),
}) 'retry_after': rate_limit_info.get('retry_after'),
'endpoint': rate_limit_info.get('endpoint'),
'expires_at': rate_limit_info.get('expires_at'),
})
return jsonify(payload)
return jsonify({'rate_limited': False}) return jsonify({'rate_limited': False})
except Exception as e: except Exception as e:
logger.error(f"Error getting Spotify rate limit status: {e}") logger.error(f"Error getting Spotify rate limit status: {e}")
@ -34378,12 +34393,12 @@ def _emit_rate_monitor_loop():
# Add Spotify rate limit state # Add Spotify rate limit state
try: try:
if spotify_client: spotify_status = get_spotify_status(spotify_client=spotify_client)
rl_info = spotify_client.get_rate_limit_info() rl_info = spotify_status.get('rate_limit')
if rl_info: if spotify_status.get('rate_limited') and rl_info:
payload['spotify']['rate_limited'] = True payload['spotify']['rate_limited'] = True
payload['spotify']['rl_remaining'] = rl_info.get('remaining_seconds', 0) payload['spotify']['rl_remaining'] = rl_info.get('remaining_seconds', 0)
payload['spotify']['rl_endpoint'] = rl_info.get('endpoint', '') payload['spotify']['rl_endpoint'] = rl_info.get('endpoint', '')
except Exception: except Exception:
pass pass