danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Make Spotify status updates event-driven

Browse source 
- move Spotify status publishing onto auth, disconnect, and rate-limit transitions
- keep dashboard and debug consumers on the shared cached snapshot
- leave only the initial snapshot seed as a fallback probe
This commit is contained in:
Antti Kettunen 2026-05-02 12:13:17 +03:00
parent cc13fb8f01
commit 36131656dd
No known key found for this signature in database
GPG key ID: C6B2A3D250359BD7
5 changed files with 253 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
core/debug_info.py
View file

@ -300,23 +300,24 @@ def get_debug_info():
# API rate monitor — current calls/min, 24h totals, peaks, rate limit events
try:
from core.api_call_tracker import api_call_tracker
from core.metadata.status import get_spotify_status
rates = api_call_tracker.get_all_rates()
info['api_rates'] = rates
# Rich 24h debug summary with peaks, totals, per-endpoint breakdown, events
info['api_debug_summary'] = api_call_tracker.get_debug_summary()
# Spotify rate limit details
if spotify_client:
rl_info = spotify_client.get_rate_limit_info()
if rl_info:
info['spotify_rate_limit'] = {
'active': True,
'remaining_seconds': rl_info.get('remaining_seconds', 0),
'retry_after': rl_info.get('retry_after', 0),
'endpoint': rl_info.get('endpoint', ''),
'expires_at': rl_info.get('expires_at', ''),
}
else:
info['spotify_rate_limit'] = {'active': False}
spotify_status = get_spotify_status(spotify_client=spotify_client)
rl_info = spotify_status.get('rate_limit')
if spotify_status.get('rate_limited') and rl_info:
info['spotify_rate_limit'] = {
'active': True,
'remaining_seconds': rl_info.get('remaining_seconds', 0),
'retry_after': rl_info.get('retry_after', 0),
'endpoint': rl_info.get('endpoint', ''),
'expires_at': rl_info.get('expires_at', ''),
}
else:
info['spotify_rate_limit'] = {'active': False}
except Exception:
info['api_rates'] = {}
info['api_debug_summary'] = {}

4
core/metadata/__init__.py
View file

@ -42,8 +42,6 @@ from core.metadata.registry import (
)
from core.metadata.status import (
METADATA_SOURCE_STATUS_TTL,
SPOTIFY_STATUS_TTL_ACTIVE,
SPOTIFY_STATUS_TTL_IDLE,
get_metadata_source_status,
get_spotify_status,
get_status_snapshot,
@ -98,7 +96,5 @@ __all__ = [
"register_profile_spotify_credentials_provider",
"register_runtime_clients",
"resolve_album_reference",
"SPOTIFY_STATUS_TTL_ACTIVE",
"SPOTIFY_STATUS_TTL_IDLE",
"invalidate_metadata_status_caches",
]

140
core/metadata/status.py
View file

@ -6,7 +6,6 @@ import threading
import time
from typing import Any, Dict, Optional
from config.settings import config_manager
from utils.logging_config import get_logger
from core.metadata.registry import get_primary_source_status
@ -14,8 +13,8 @@ from core.metadata.registry import get_primary_source_status
logger = get_logger("metadata.status")
METADATA_SOURCE_STATUS_TTL = 120
SPOTIFY_STATUS_TTL_ACTIVE = 15
SPOTIFY_STATUS_TTL_IDLE = 300
_UNSET = object()
_status_lock = threading.RLock()
_metadata_source_status_cache: Dict[str, Any] = {
@ -32,21 +31,80 @@ _spotify_status_cache: Dict[str, Any] = {
"post_ban_cooldown": None,
}
_spotify_status_timestamp = 0.0
def _get_config_value(key: str, default: Any = None) -> Any:
try:
return config_manager.get(key, default)
except Exception:
return default
_spotify_status_initialized = False
_spotify_rate_limit_expires_at = 0.0
_spotify_post_ban_cooldown_expires_at = 0.0
def invalidate_metadata_status_caches() -> None:
"""Mark the cached metadata-source and Spotify status snapshots stale."""
global _metadata_source_status_timestamp, _spotify_status_timestamp
"""Mark the cached metadata-source snapshot stale."""
global _metadata_source_status_timestamp
with _status_lock:
_metadata_source_status_timestamp = 0.0
_spotify_status_timestamp = 0.0
def publish_spotify_status(
*,
connected: Any = _UNSET,
authenticated: Any = _UNSET,
rate_limited: Any = _UNSET,
rate_limit: Any = _UNSET,
post_ban_cooldown: Any = _UNSET,
) -> Dict[str, Any]:
"""Update the cached Spotify status snapshot from an event."""
global _spotify_status_timestamp, _spotify_status_initialized
global _spotify_rate_limit_expires_at, _spotify_post_ban_cooldown_expires_at
with _status_lock:
if connected is not _UNSET:
_spotify_status_cache["connected"] = connected
if authenticated is not _UNSET:
_spotify_status_cache["authenticated"] = authenticated
if rate_limited is not _UNSET:
_spotify_status_cache["rate_limited"] = rate_limited
if rate_limit is not _UNSET:
_spotify_status_cache["rate_limit"] = rate_limit
if rate_limit and isinstance(rate_limit, dict):
_spotify_rate_limit_expires_at = float(rate_limit.get("expires_at") or 0.0)
else:
_spotify_rate_limit_expires_at = 0.0
if post_ban_cooldown is not _UNSET:
_spotify_status_cache["post_ban_cooldown"] = post_ban_cooldown
if post_ban_cooldown is not None:
_spotify_post_ban_cooldown_expires_at = time.time() + max(0, float(post_ban_cooldown))
else:
_spotify_post_ban_cooldown_expires_at = 0.0
_spotify_status_timestamp = time.time()
_spotify_status_initialized = True
_normalize_spotify_status_locked(_spotify_status_timestamp)
return dict(_spotify_status_cache)
def refresh_spotify_status_from_client(spotify_client: Optional[Any]) -> Dict[str, Any]:
"""Probe Spotify once to seed the cache when no event has populated it yet."""
if spotify_client is None:
with _status_lock:
return dict(_spotify_status_cache)
try:
is_rate_limited = spotify_client.is_rate_limited() if spotify_client else False
rate_limit_info = spotify_client.get_rate_limit_info() if (spotify_client and is_rate_limited) else None
cooldown_remaining = spotify_client.get_post_ban_cooldown_remaining() if spotify_client else 0
authenticated = spotify_client.is_spotify_authenticated() if spotify_client else False
except Exception as exc:
logger.debug("Spotify status probe failed: %s", exc)
authenticated = False
is_rate_limited = False
rate_limit_info = None
cooldown_remaining = 0
return publish_spotify_status(
connected=authenticated,
authenticated=authenticated,
rate_limited=is_rate_limited,
rate_limit=rate_limit_info,
post_ban_cooldown=cooldown_remaining if cooldown_remaining > 0 else None,
)
def get_metadata_source_status() -> Dict[str, Any]:
@ -75,36 +133,42 @@ def get_metadata_source_status() -> Dict[str, Any]:
def get_spotify_status(spotify_client: Optional[Any] = None) -> Dict[str, Any]:
"""Return a cached Spotify-specific status snapshot."""
global _spotify_status_timestamp
current_time = time.time()
configured_source = _get_config_value("metadata.fallback_source", "deezer") or "deezer"
ttl = SPOTIFY_STATUS_TTL_ACTIVE if configured_source == "spotify" else SPOTIFY_STATUS_TTL_IDLE
with _status_lock:
if _spotify_status_timestamp and current_time - _spotify_status_timestamp <= ttl:
if _spotify_status_initialized:
_normalize_spotify_status_locked(time.time())
return dict(_spotify_status_cache)
try:
is_rate_limited = spotify_client.is_rate_limited() if spotify_client else False
rate_limit_info = spotify_client.get_rate_limit_info() if (spotify_client and is_rate_limited) else None
cooldown_remaining = spotify_client.get_post_ban_cooldown_remaining() if spotify_client else 0
authenticated = spotify_client.is_spotify_authenticated() if spotify_client else False
return refresh_spotify_status_from_client(spotify_client)
with _status_lock:
_spotify_status_cache.update({
"connected": authenticated,
"authenticated": authenticated,
"rate_limited": is_rate_limited,
"rate_limit": rate_limit_info,
"post_ban_cooldown": cooldown_remaining if cooldown_remaining > 0 else None,
})
_spotify_status_timestamp = current_time
except Exception as exc:
logger.debug("Spotify status refresh failed: %s", exc)
with _status_lock:
return dict(_spotify_status_cache)
def _normalize_spotify_status_locked(current_time: float) -> None:
"""Update derived Spotify status fields and clear expired ban state."""
global _spotify_rate_limit_expires_at, _spotify_post_ban_cooldown_expires_at
rate_limit = _spotify_status_cache.get("rate_limit")
if _spotify_status_cache.get("rate_limited") and rate_limit and isinstance(rate_limit, dict):
expires_at = float(rate_limit.get("expires_at") or 0.0)
if expires_at > 0:
remaining = int(max(0, expires_at - current_time))
if remaining > 0:
_spotify_status_cache["rate_limit"] = {**rate_limit, "remaining_seconds": remaining}
_spotify_rate_limit_expires_at = expires_at
else:
_spotify_status_cache["rate_limited"] = False
_spotify_status_cache["rate_limit"] = None
_spotify_rate_limit_expires_at = 0.0
elif _spotify_rate_limit_expires_at and current_time >= _spotify_rate_limit_expires_at:
_spotify_status_cache["rate_limited"] = False
_spotify_status_cache["rate_limit"] = None
_spotify_rate_limit_expires_at = 0.0
if _spotify_post_ban_cooldown_expires_at > 0:
remaining = int(max(0, _spotify_post_ban_cooldown_expires_at - current_time))
if remaining > 0:
_spotify_status_cache["post_ban_cooldown"] = remaining
else:
_spotify_status_cache["post_ban_cooldown"] = None
_spotify_post_ban_cooldown_expires_at = 0.0
def get_status_snapshot(spotify_client: Optional[Any] = None) -> Dict[str, Any]:

109
core/spotify_client.py
View file

@ -138,6 +138,18 @@ def _set_global_rate_limit(retry_after_seconds, endpoint_name, has_real_header=F
)
except Exception:
pass
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=True,
rate_limited=True,
rate_limit=_get_rate_limit_info(),
post_ban_cooldown=_get_post_ban_cooldown_remaining() or None,
)
except Exception:
pass
def _is_globally_rate_limited():
@ -210,6 +222,16 @@ def _clear_rate_limit():
_rate_limit_hit_count = 0
_rate_limit_first_hit = 0
logger.info("Global rate limit ban cleared (including post-ban cooldown)")
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
rate_limited=False,
rate_limit=None,
post_ban_cooldown=None,
)
except Exception:
pass
def _detect_and_set_rate_limit(exception, endpoint_name="unknown"):
@ -603,13 +625,38 @@ class SpotifyClient:
"""Check if Spotify client is specifically authenticated (not just iTunes fallback).
Results are cached for 60 seconds to avoid excessive API calls.
During rate limit bans and post-ban cooldown, returns False without making API calls."""
rate_limited_state = False
if self.sp is None:
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=False,
rate_limited=_is_globally_rate_limited(),
rate_limit=_get_rate_limit_info(),
post_ban_cooldown=_get_post_ban_cooldown_remaining() or None,
)
except Exception:
pass
return False
# If globally rate limited, report as NOT authenticated so callers
# skip Spotify and fall through to iTunes fallback naturally.
# This prevents any API calls that could extend the ban.
if _is_globally_rate_limited():
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=True,
rate_limited=True,
rate_limit=_get_rate_limit_info(),
post_ban_cooldown=_get_post_ban_cooldown_remaining() or None,
)
except Exception:
pass
return False
# Post-ban cooldown: after a ban expires, don't probe Spotify immediately.
@ -618,11 +665,35 @@ class SpotifyClient:
if _is_in_post_ban_cooldown():
remaining = _get_post_ban_cooldown_remaining()
logger.debug(f"Post-ban cooldown active ({remaining}s left), skipping auth probe")
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=True,
rate_limited=False,
rate_limit=None,
post_ban_cooldown=remaining or None,
)
except Exception:
pass
return False
# Check cache first (lock only for brief read)
with self._auth_cache_lock:
if self._auth_cached_result is not None and (time.time() - self._auth_cache_time) < self._AUTH_CACHE_TTL:
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=self._auth_cached_result,
authenticated=self._auth_cached_result,
rate_limited=False,
rate_limit=None,
post_ban_cooldown=None,
)
except Exception:
pass
return self._auth_cached_result
# Cache miss — make API call outside the lock.
@ -637,6 +708,18 @@ class SpotifyClient:
with self._auth_cache_lock:
self._auth_cached_result = False
self._auth_cache_time = time.time()
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=False,
rate_limited=False,
rate_limit=None,
post_ban_cooldown=None,
)
except Exception:
pass
return False
except Exception:
pass
@ -668,6 +751,7 @@ class SpotifyClient:
ban_duration = max(delay, _BASE_UNKNOWN_BAN)
_set_global_rate_limit(ban_duration, 'is_spotify_authenticated', has_real_header=has_real_header)
logger.warning(f"Auth probe rate limited — activating {ban_duration}s global ban")
rate_limited_state = True
result = True
else:
logger.debug(f"Spotify authentication check failed: {e}")
@ -677,6 +761,19 @@ class SpotifyClient:
self._auth_cached_result = result
self._auth_cache_time = time.time()
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=result,
authenticated=result,
rate_limited=rate_limited_state,
rate_limit=_get_rate_limit_info() if rate_limited_state else None,
post_ban_cooldown=None,
)
except Exception:
pass
return result
def disconnect(self):
@ -686,6 +783,18 @@ class SpotifyClient:
self.user_id = None
self._invalidate_auth_cache()
_clear_rate_limit()
try:
from core.metadata.status import publish_spotify_status
publish_spotify_status(
connected=False,
authenticated=False,
rate_limited=False,
rate_limit=None,
post_ban_cooldown=None,
)
except Exception:
pass
cache_path = 'config/.spotify_cache'
try:

43
web_server.py
View file

@ -105,6 +105,8 @@ from core.metadata.registry import (
)
from core.metadata.status import (
get_status_snapshot as get_metadata_status_snapshot,
get_spotify_status,
publish_spotify_status,
invalidate_metadata_status_caches,
)
from core.imports.context import (
@ -4152,6 +4154,14 @@ def handle_settings():
genius_worker._init_client()
if tidal_enrichment_worker:
tidal_enrichment_worker.client = tidal_client
if 'spotify' in new_settings:
publish_spotify_status(
connected=False,
authenticated=False,
rate_limited=False,
rate_limit=None,
post_ban_cooldown=None,
)
# Invalidate status cache so next poll reflects new settings (e.g. fallback source change)
invalidate_metadata_status_caches()
logger.info("Service clients re-initialized with new settings.")
@ -5906,15 +5916,20 @@ def spotify_disconnect():
def spotify_rate_limit_status():
"""Get Spotify rate limit ban details"""
try:
info = spotify_client.get_rate_limit_info()
info = get_spotify_status(spotify_client=spotify_client)
rate_limit_info = info.get('rate_limit')
if info:
return jsonify({
'rate_limited': True,
'remaining_seconds': info['remaining_seconds'],
'retry_after': info['retry_after'],
'endpoint': info['endpoint'],
'expires_at': info['expires_at']
})
payload = {
'rate_limited': bool(info.get('rate_limited')),
}
if rate_limit_info:
payload.update({
'remaining_seconds': rate_limit_info.get('remaining_seconds', 0),
'retry_after': rate_limit_info.get('retry_after'),
'endpoint': rate_limit_info.get('endpoint'),
'expires_at': rate_limit_info.get('expires_at'),
})
return jsonify(payload)
return jsonify({'rate_limited': False})
except Exception as e:
logger.error(f"Error getting Spotify rate limit status: {e}")
@ -34378,12 +34393,12 @@ def _emit_rate_monitor_loop():
# Add Spotify rate limit state
try:
if spotify_client:
rl_info = spotify_client.get_rate_limit_info()
if rl_info:
payload['spotify']['rate_limited'] = True
payload['spotify']['rl_remaining'] = rl_info.get('remaining_seconds', 0)
payload['spotify']['rl_endpoint'] = rl_info.get('endpoint', '')
spotify_status = get_spotify_status(spotify_client=spotify_client)
rl_info = spotify_status.get('rate_limit')
if spotify_status.get('rate_limited') and rl_info:
payload['spotify']['rate_limited'] = True
payload['spotify']['rl_remaining'] = rl_info.get('remaining_seconds', 0)
payload['spotify']['rl_endpoint'] = rl_info.get('endpoint', '')
except Exception:
pass