danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #303 from kettui/fix/watchlist-scanner-fixes

Browse source 
Consolidate watchlist scanning code, respect primary metadata provider
This commit is contained in:
BoulderBadgeDad 2026-04-16 08:29:22 -07:00 committed by GitHub
commit 4dab3de2d6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 1444 additions and 1201 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

270
core/spotify_client.py
View file

@ -1098,82 +1098,85 @@ class SpotifyClient:
return []
@rate_limited
def search_tracks(self, query: str, limit: int = 10) -> List[Track]:
"""Search for tracks - falls back to configured metadata source if Spotify not authenticated"""
def search_tracks(self, query: str, limit: int = 10, allow_fallback: bool = True) -> List[Track]:
"""Search for tracks.
When allow_fallback is True, falls back to the configured metadata source
if Spotify is unavailable or returns an error.
"""
cache = get_metadata_cache()
effective_limit = min(limit, 50) # Spotify API max is 50
# Check Spotify cache first so cached data remains usable even when
# Spotify is temporarily unavailable or rate limited.
cached_results = cache.get_search_results('spotify', 'track', query, effective_limit)
if cached_results is not None:
tracks = []
for raw in cached_results:
try:
tracks.append(Track.from_spotify_track(raw))
except Exception:
pass
if tracks:
return tracks
use_spotify = self.is_spotify_authenticated()
if use_spotify:
# Check Spotify cache
effective_limit = min(limit, 50) # Spotify API max is 50
cached_results = cache.get_search_results('spotify', 'track', query, effective_limit)
if cached_results is not None:
try:
results = self.sp.search(q=query, type='track', limit=effective_limit)
tracks = []
for raw in cached_results:
try:
tracks.append(Track.from_spotify_track(raw))
except Exception:
pass
if tracks:
return tracks
raw_items = results['tracks']['items']
# Skip Spotify if globally rate limited — fall through to fallback
if self.is_rate_limited():
logger.debug(f"Spotify rate limited, skipping track search for: {query}")
use_spotify = False
else:
try:
results = self.sp.search(q=query, type='track', limit=effective_limit)
tracks = []
raw_items = results['tracks']['items']
for track_data in raw_items:
track = Track.from_spotify_track(track_data)
tracks.append(track)
for track_data in raw_items:
track = Track.from_spotify_track(track_data)
tracks.append(track)
# Cache individual tracks + search mapping
entries = [(td.get('id'), td) for td in raw_items if td.get('id')]
if entries:
cache.store_entities_bulk('spotify', 'track', entries)
cache.store_search_results('spotify', 'track', query, effective_limit,
[td.get('id') for td in raw_items if td.get('id')])
# Cache individual tracks + search mapping
entries = [(td.get('id'), td) for td in raw_items if td.get('id')]
if entries:
cache.store_entities_bulk('spotify', 'track', entries)
cache.store_search_results('spotify', 'track', query, effective_limit,
[td.get('id') for td in raw_items if td.get('id')])
return tracks
return tracks
except Exception as e:
_detect_and_set_rate_limit(e, 'search_tracks')
logger.error(f"Error searching tracks via Spotify: {e}")
# Fall through to fallback
except Exception as e:
_detect_and_set_rate_limit(e, 'search_tracks')
logger.error(f"Error searching tracks via Spotify: {e}")
# Fall through to fallback
# Fallback (iTunes or Deezer — configured in settings)
logger.debug(f"Using {self._fallback_source} fallback for track search: {query}")
return self._fallback.search_tracks(query, limit)
if allow_fallback:
logger.debug(f"Using {self._fallback_source} fallback for track search: {query}")
return self._fallback.search_tracks(query, limit)
return []
@rate_limited
def search_artists(self, query: str, limit: int = 10) -> List[Artist]:
"""Search for artists - falls back to configured metadata source if Spotify not authenticated"""
def search_artists(self, query: str, limit: int = 10, allow_fallback: bool = True) -> List[Artist]:
"""Search for artists.
When allow_fallback is True, falls back to the configured metadata source
if Spotify is unavailable or returns an error.
"""
cache = get_metadata_cache()
# Check Spotify cache first so cached data remains usable even when
# Spotify is temporarily unavailable or rate limited.
cached_results = cache.get_search_results('spotify', 'artist', query, min(limit, 10))
if cached_results is not None:
artists = []
for raw in cached_results:
try:
artists.append(Artist.from_spotify_artist(raw))
except Exception:
pass
if artists:
query_lower = query.lower().strip()
artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1))
return artists
use_spotify = self.is_spotify_authenticated()
if use_spotify:
# Check Spotify cache
cached_results = cache.get_search_results('spotify', 'artist', query, min(limit, 10))
if cached_results is not None:
artists = []
for raw in cached_results:
try:
artists.append(Artist.from_spotify_artist(raw))
except Exception:
pass
if artists:
query_lower = query.lower().strip()
artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1))
return artists
if self.is_rate_limited():
logger.debug(f"Spotify rate limited, skipping artist search for: {query}")
use_spotify = False
if use_spotify:
try:
search_query = f'artist:{query}' if len(query.strip()) <= 4 else query
@ -1204,67 +1207,74 @@ class SpotifyClient:
# Fall through to iTunes fallback
# Fallback (iTunes or Deezer)
logger.debug(f"Using {self._fallback_source} fallback for artist search: {query}")
artists = self._fallback.search_artists(query, limit)
query_lower = query.lower().strip()
artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1))
return artists
if allow_fallback:
logger.debug(f"Using {self._fallback_source} fallback for artist search: {query}")
artists = self._fallback.search_artists(query, limit)
query_lower = query.lower().strip()
artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1))
return artists
return []
@rate_limited
def search_albums(self, query: str, limit: int = 10) -> List[Album]:
"""Search for albums - falls back to configured metadata source if Spotify not authenticated"""
def search_albums(self, query: str, limit: int = 10, allow_fallback: bool = True) -> List[Album]:
"""Search for albums.
When allow_fallback is True, falls back to the configured metadata source
if Spotify is unavailable or returns an error.
"""
cache = get_metadata_cache()
# Check Spotify cache first so cached data remains usable even when
# Spotify is temporarily unavailable or rate limited.
cached_results = cache.get_search_results('spotify', 'album', query, min(limit, 10))
if cached_results is not None:
albums = []
for raw in cached_results:
try:
albums.append(Album.from_spotify_album(raw))
except Exception:
pass
if albums:
return albums
use_spotify = self.is_spotify_authenticated()
if use_spotify:
# Check Spotify cache
cached_results = cache.get_search_results('spotify', 'album', query, min(limit, 10))
if cached_results is not None:
try:
results = self.sp.search(q=query, type='album', limit=min(limit, 10))
albums = []
for raw in cached_results:
try:
albums.append(Album.from_spotify_album(raw))
except Exception:
pass
if albums:
return albums
raw_items = results['albums']['items']
if use_spotify:
# Skip Spotify if globally rate limited — fall through to fallback
if self.is_rate_limited():
logger.debug(f"Spotify rate limited, skipping album search for: {query}")
use_spotify = False
else:
try:
results = self.sp.search(q=query, type='album', limit=min(limit, 10))
albums = []
raw_items = results['albums']['items']
for album_data in raw_items:
album = Album.from_spotify_album(album_data)
albums.append(album)
for album_data in raw_items:
album = Album.from_spotify_album(album_data)
albums.append(album)
# Cache individual albums + search mapping (skip if full data already cached)
entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')]
if entries:
cache.store_entities_bulk('spotify', 'album', entries, skip_if_exists=True)
cache.store_search_results('spotify', 'album', query, min(limit, 10),
[ad.get('id') for ad in raw_items if ad.get('id')])
# Cache individual albums + search mapping (skip if full data already cached)
entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')]
if entries:
cache.store_entities_bulk('spotify', 'album', entries, skip_if_exists=True)
cache.store_search_results('spotify', 'album', query, min(limit, 10),
[ad.get('id') for ad in raw_items if ad.get('id')])
return albums
return albums
except Exception as e:
_detect_and_set_rate_limit(e, 'search_albums')
logger.error(f"Error searching albums via Spotify: {e}")
# Fall through to iTunes fallback
except Exception as e:
_detect_and_set_rate_limit(e, 'search_albums')
logger.error(f"Error searching albums via Spotify: {e}")
# Fall through to iTunes fallback
# Fallback (iTunes or Deezer)
logger.debug(f"Using {self._fallback_source} fallback for album search: {query}")
return self._fallback.search_albums(query, limit)
if allow_fallback:
logger.debug(f"Using {self._fallback_source} fallback for album search: {query}")
return self._fallback.search_albums(query, limit)
return []
@rate_limited
def get_track_details(self, track_id: str) -> Optional[Dict[str, Any]]:
"""Get detailed track information - falls back to configured metadata source"""
def get_track_details(self, track_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]:
"""Get detailed track information.
When allow_fallback is True, falls back to the configured metadata source
for non-Spotify IDs or Spotify failure.
"""
# Check cache — we store raw track_data, reconstruct enhanced on hit
cache = get_metadata_cache()
fallback_src = self._fallback_source
@ -1277,7 +1287,7 @@ class SpotifyClient:
return self._build_enhanced_track(cached)
# Simplified track cached by get_album_tracks — treat as cache miss
logger.debug(f"Cache hit for track {track_id} lacks album data, fetching full data")
else:
elif allow_fallback:
# Fallback cache hit — delegate to fallback client which reconstructs enhanced format
return self._fallback.get_track_details(track_id)
@ -1298,7 +1308,7 @@ class SpotifyClient:
# Fall through to iTunes fallback
# Fallback - only if ID is numeric (non-Spotify format)
if self._is_itunes_id(track_id):
if allow_fallback and self._is_itunes_id(track_id):
logger.debug(f"Using {fallback_src} fallback for track details: {track_id}")
result = self._fallback.get_track_details(track_id)
return result
@ -1354,8 +1364,12 @@ class SpotifyClient:
return None
@rate_limited
def get_album(self, album_id: str) -> Optional[Dict[str, Any]]:
"""Get album information - falls back to configured metadata source"""
def get_album(self, album_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]:
"""Get album information.
When allow_fallback is True, falls back to the configured metadata source
for non-Spotify IDs or Spotify failure.
"""
# Check cache first
cache = get_metadata_cache()
fallback_src = self._fallback_source
@ -1368,7 +1382,7 @@ class SpotifyClient:
return cached
# Simplified album cached by get_artist_albums — treat as cache miss
logger.debug(f"Cache hit for album {album_id} lacks tracks, fetching full data")
else:
elif allow_fallback:
# Fallback cache hit — delegate to fallback client
return self._fallback.get_album(album_id)
@ -1385,7 +1399,7 @@ class SpotifyClient:
# Fall through to fallback
# Fallback - only if ID is numeric (non-Spotify format)
if self._is_itunes_id(album_id):
if allow_fallback and self._is_itunes_id(album_id):
logger.debug(f"Using {fallback_src} fallback for album: {album_id}")
return self._fallback.get_album(album_id)
else:
@ -1393,8 +1407,12 @@ class SpotifyClient:
return None
@rate_limited
def get_album_tracks(self, album_id: str) -> Optional[Dict[str, Any]]:
"""Get album tracks - falls back to configured metadata source"""
def get_album_tracks(self, album_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]:
"""Get album tracks.
When allow_fallback is True, falls back to the configured metadata source
for non-Spotify IDs or Spotify failure.
"""
# Cache key uses album_id with '_tracks' suffix to differentiate from album metadata
cache = get_metadata_cache()
fallback_src = self._fallback_source
@ -1458,7 +1476,7 @@ class SpotifyClient:
# Fall through to iTunes fallback
# Fallback - only if ID is numeric (non-Spotify format)
if self._is_itunes_id(album_id):
if allow_fallback and self._is_itunes_id(album_id):
logger.debug(f"Using {fallback_src} fallback for album tracks: {album_id}")
result = self._fallback.get_album_tracks(album_id)
return result
@ -1467,8 +1485,12 @@ class SpotifyClient:
return None
@rate_limited
def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 10, skip_cache: bool = False, max_pages: int = 0) -> List[Album]:
"""Get albums by artist ID - falls back to iTunes if Spotify not authenticated.
def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 10,
skip_cache: bool = False, max_pages: int = 0, allow_fallback: bool = True) -> List[Album]:
"""Get albums by artist ID.
When allow_fallback is True, falls back to iTunes/Deezer if Spotify
is not authenticated or errors.
Set skip_cache=True for watchlist scans that need fresh data to detect new releases.
Set max_pages to limit pagination (0 = fetch all). Spotify returns newest first,
so max_pages=1 is sufficient for new release detection."""
@ -1540,7 +1562,7 @@ class SpotifyClient:
# Fall through to iTunes fallback
# Fallback - only if ID is numeric (non-Spotify format)
if self._is_itunes_id(artist_id):
if allow_fallback and self._is_itunes_id(artist_id):
logger.debug(f"Using {fallback_src} fallback for artist albums: {artist_id}")
return self._fallback.get_artist_albums(artist_id, album_type, limit)
else:
@ -1559,9 +1581,9 @@ class SpotifyClient:
return None
@rate_limited
def get_artist(self, artist_id: str) -> Optional[Dict[str, Any]]:
def get_artist(self, artist_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]:
"""
Get full artist details - falls back to configured metadata source.
Get full artist details.
Args:
artist_id: Artist ID (Spotify or fallback source depending on authentication)
@ -1577,8 +1599,10 @@ class SpotifyClient:
if cached:
if source == 'spotify':
return cached # Spotify raw format is the expected format
# Fallback cache hit — delegate to fallback client which reconstructs Spotify-compatible format
return self._fallback.get_artist(artist_id)
if allow_fallback:
# Fallback cache hit — delegate to fallback client which reconstructs Spotify-compatible format
return self._fallback.get_artist(artist_id)
return None
if self.is_spotify_authenticated():
try:
@ -1592,7 +1616,7 @@ class SpotifyClient:
# Fall through to iTunes fallback
# Fallback - only if ID is numeric (non-Spotify format)
if self._is_itunes_id(artist_id):
if allow_fallback and self._is_itunes_id(artist_id):
logger.debug(f"Using {fallback_src} fallback for artist: {artist_id}")
return self._fallback.get_artist(artist_id)
else:

1266
core/watchlist_scanner.py
View file

File diff suppressed because it is too large Load diff

401
tests/test_watchlist_scanner_scan.py Normal file
View file

@ -0,0 +1,401 @@
import sys
import types
if "spotipy" not in sys.modules:
spotipy = types.ModuleType("spotipy")
class _DummySpotify:
def __init__(self, *args, **kwargs):
pass
oauth2 = types.ModuleType("spotipy.oauth2")
class _DummyOAuth:
def __init__(self, *args, **kwargs):
pass
spotipy.Spotify = _DummySpotify
oauth2.SpotifyOAuth = _DummyOAuth
oauth2.SpotifyClientCredentials = _DummyOAuth
spotipy.oauth2 = oauth2
sys.modules["spotipy"] = spotipy
sys.modules["spotipy.oauth2"] = oauth2
if "config.settings" not in sys.modules:
config_pkg = types.ModuleType("config")
settings_mod = types.ModuleType("config.settings")
class _DummyConfigManager:
def get(self, key, default=None):
return default
def get_active_media_server(self):
return "plex"
settings_mod.config_manager = _DummyConfigManager()
config_pkg.settings = settings_mod
sys.modules["config"] = config_pkg
sys.modules["config.settings"] = settings_mod
if "core.matching_engine" not in sys.modules:
matching_engine_mod = types.ModuleType("core.matching_engine")
class _DummyMatchingEngine:
def clean_title(self, title):
return title
matching_engine_mod.MusicMatchingEngine = _DummyMatchingEngine
sys.modules["core.matching_engine"] = matching_engine_mod
import core.watchlist_scanner as watchlist_scanner_module
from core.watchlist_scanner import WatchlistScanner
class _FakeSpotifyClient:
def __init__(self, search_results=None):
self.search_results = list(search_results or [])
self.search_calls = []
def is_spotify_authenticated(self):
return False
def search_artists(self, query, limit=1, allow_fallback=True):
self.search_calls.append((query, limit, allow_fallback))
return list(self.search_results) if allow_fallback else []
class _FakeMetadataService:
def __init__(self, album_data, spotify_client=None):
self.spotify = spotify_client or _FakeSpotifyClient()
self.itunes = types.SimpleNamespace()
self._album_data = album_data
def get_album(self, album_id):
return self._album_data
class _FakeSourceClient:
def __init__(self, *, artist_id: str, albums, image_url: str):
self.artist_id = artist_id
self.albums = list(albums)
self.image_url = image_url
self.search_calls = []
self.album_calls = []
self.artist_calls = []
def search_artists(self, query, limit=1, **kwargs):
self.search_calls.append((query, limit, kwargs))
return [types.SimpleNamespace(id=self.artist_id, name=query)]
def get_artist_albums(self, artist_id, album_type='album,single', limit=50, **kwargs):
self.album_calls.append((artist_id, album_type, limit, kwargs))
return list(self.albums)
def get_artist(self, artist_id):
self.artist_calls.append(artist_id)
return {
"id": artist_id,
"images": [{"url": self.image_url}] if self.image_url else [],
}
class _FakeDB:
def __init__(self, artists):
self.artists = artists
self.similar_calls = []
def get_watchlist_artists(self, profile_id=None):
return list(self.artists)
def has_fresh_similar_artists(self, *args, **kwargs):
self.similar_calls.append((args, kwargs))
return False
def _build_artist(name="Artist One", profile_id=11):
return types.SimpleNamespace(
artist_name=name,
spotify_artist_id="sp-artist",
itunes_artist_id="it-artist",
deezer_artist_id="dz-artist",
discogs_artist_id="dg-artist",
last_scan_timestamp=None,
id=123,
profile_id=profile_id,
include_albums=True,
include_eps=True,
include_singles=True,
include_live=False,
include_remixes=False,
include_acoustic=False,
include_compilations=False,
include_instrumentals=False,
lookback_days=7,
image_url=None,
)
def _build_scanner(album_data, artists):
scanner = WatchlistScanner(metadata_service=_FakeMetadataService(album_data))
scanner._database = _FakeDB(artists)
scanner._wishlist_service = types.SimpleNamespace()
scanner._matching_engine = types.SimpleNamespace()
return scanner
def test_scan_watchlist_profile_loads_artists_and_applies_overrides(monkeypatch):
artist = _build_artist()
scanner = _build_scanner({"tracks": {"items": []}}, [artist])
loaded_profiles = []
override_calls = []
scan_calls = []
monkeypatch.setattr(scanner.database, "get_watchlist_artists", lambda profile_id=None: loaded_profiles.append(profile_id) or [artist])
monkeypatch.setattr(scanner, "_apply_global_watchlist_overrides", lambda artists: override_calls.append(list(artists)))
monkeypatch.setattr(scanner, "scan_watchlist_artists", lambda artists, **kwargs: scan_calls.append((list(artists), kwargs)) or ["ok"])
result = scanner.scan_watchlist_profile(42)
assert result == ["ok"]
assert loaded_profiles == [42]
assert override_calls and override_calls[0][0].artist_name == "Artist One"
assert scan_calls and scan_calls[0][0][0].artist_name == "Artist One"
assert scan_calls[0][1]["profile_id"] == 42
def test_scan_watchlist_artists_scans_tracks_and_updates_state(monkeypatch):
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0)
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0)
artist = _build_artist()
album = types.SimpleNamespace(id="album-1", name="Album One")
album_data = {
"name": "Album One",
"images": [{"url": "https://example.com/album.jpg"}],
"tracks": {
"items": [
{
"id": "track-1",
"name": "Track One",
"track_number": 1,
"disc_number": 1,
"artists": [{"name": "Artist One"}],
}
]
},
}
scanner = _build_scanner(album_data, [artist])
scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False
monkeypatch.setattr(scanner, "_backfill_missing_ids", lambda *args, **kwargs: None)
monkeypatch.setattr(scanner, "get_artist_image_url", lambda *_args, **_kwargs: "https://example.com/artist.jpg")
monkeypatch.setattr(scanner, "get_artist_discography_for_watchlist", lambda *_args, **_kwargs: [album])
monkeypatch.setattr(scanner, "_get_lookback_period_setting", lambda: "30")
monkeypatch.setattr(scanner, "_get_rescan_cutoff", lambda: None)
monkeypatch.setattr(scanner, "_should_include_release", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "_should_include_track", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "is_track_missing_from_library", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0)
scan_state = {}
results = scanner.scan_watchlist_artists([artist], scan_state=scan_state)
assert len(results) == 1
assert results[0].success is True
assert results[0].new_tracks_found == 1
assert results[0].tracks_added_to_wishlist == 1
assert scan_state["status"] == "completed"
assert scan_state["summary"]["successful_scans"] == 1
assert scan_state["summary"]["new_tracks_found"] == 1
assert scan_state["summary"]["tracks_added_to_wishlist"] == 1
assert scan_state["recent_wishlist_additions"][0]["track_name"] == "Track One"
def test_scan_watchlist_artists_skips_placeholder_tracklists(monkeypatch):
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0)
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0)
artist = _build_artist()
album = types.SimpleNamespace(id="album-1", name="Album One")
album_data = {
"name": "Album One",
"images": [{"url": "https://example.com/album.jpg"}],
"tracks": {
"items": [
{
"id": "track-1",
"name": "Track 1",
"track_number": 1,
"disc_number": 1,
"artists": [{"name": "Artist One"}],
},
{
"id": "track-2",
"name": "Track 2",
"track_number": 2,
"disc_number": 1,
"artists": [{"name": "Artist One"}],
},
]
},
}
scanner = _build_scanner(album_data, [artist])
scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False
monkeypatch.setattr(scanner, "_backfill_missing_ids", lambda *args, **kwargs: None)
monkeypatch.setattr(scanner, "get_artist_image_url", lambda *_args, **_kwargs: "https://example.com/artist.jpg")
monkeypatch.setattr(scanner, "get_artist_discography_for_watchlist", lambda *_args, **_kwargs: [album])
monkeypatch.setattr(scanner, "_get_lookback_period_setting", lambda: "30")
monkeypatch.setattr(scanner, "_get_rescan_cutoff", lambda: None)
monkeypatch.setattr(scanner, "_should_include_release", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "_should_include_track", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "is_track_missing_from_library", lambda *_args, **_kwargs: True)
add_calls = []
monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *args, **kwargs: add_calls.append((args, kwargs)) or True)
monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0)
scan_state = {}
results = scanner.scan_watchlist_artists([artist], scan_state=scan_state)
assert len(results) == 1
assert results[0].success is True
assert results[0].new_tracks_found == 0
assert results[0].tracks_added_to_wishlist == 0
assert add_calls == []
assert scan_state["summary"]["new_tracks_found"] == 0
assert scan_state["summary"]["tracks_added_to_wishlist"] == 0
def test_scan_watchlist_artists_honors_cancel_check(monkeypatch):
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0)
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0)
artist_a = _build_artist("Artist One")
artist_b = _build_artist("Artist Two")
album = types.SimpleNamespace(id="album-1", name="Album One")
album_data = {
"name": "Album One",
"tracks": {"items": []},
}
scanner = _build_scanner(album_data, [artist_a, artist_b])
scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False
monkeypatch.setattr(scanner, "_backfill_missing_ids", lambda *args, **kwargs: None)
monkeypatch.setattr(scanner, "get_artist_image_url", lambda *_args, **_kwargs: "https://example.com/artist.jpg")
monkeypatch.setattr(scanner, "get_artist_discography_for_watchlist", lambda *_args, **_kwargs: [album])
monkeypatch.setattr(scanner, "_get_lookback_period_setting", lambda: "30")
monkeypatch.setattr(scanner, "_get_rescan_cutoff", lambda: None)
monkeypatch.setattr(scanner, "_should_include_release", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "_should_include_track", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "is_track_missing_from_library", lambda *_args, **_kwargs: False)
monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *_args, **_kwargs: False)
monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True)
monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0)
cancels = iter([False, True])
scan_state = {}
results = scanner.scan_watchlist_artists(
[artist_a, artist_b],
scan_state=scan_state,
cancel_check=lambda: next(cancels),
)
assert len(results) == 1
assert results[0].artist_name == "Artist One"
assert scan_state["status"] == "cancelled"
assert scan_state["summary"]["cancelled"] is True
assert scan_state["summary"]["successful_scans"] == 1
def test_get_artist_discography_for_watchlist_prefers_primary_source(monkeypatch):
monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None))
monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
deezer_album = types.SimpleNamespace(id="dz-album", name="Deezer Album", release_date=None)
spotify_album = types.SimpleNamespace(id="sp-album", name="Spotify Album", release_date=None)
deezer_client = _FakeSourceClient(artist_id="dz-artist", albums=[deezer_album], image_url="https://example.com/deezer.jpg")
spotify_client = _FakeSourceClient(artist_id="sp-artist", albums=[spotify_album], image_url="https://example.com/spotify.jpg")
def fake_get_client_for_source(source):
return {"deezer": deezer_client, "spotify": spotify_client}.get(source)
monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source)
artist = _build_artist()
artist.spotify_artist_id = "sp-artist"
artist.deezer_artist_id = "dz-artist"
scanner = _build_scanner({"tracks": {"items": []}}, [artist])
scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False
scanner._get_lookback_period_setting = lambda: "30"
scanner._get_rescan_cutoff = lambda: None
result = scanner.get_artist_discography_for_watchlist(artist, None)
assert result is not None
assert result.source == "deezer"
assert result.artist_id == "dz-artist"
assert result.albums and result.albums[0].id == "dz-album"
assert deezer_client.album_calls
assert spotify_client.album_calls == []
def test_get_artist_discography_for_watchlist_falls_back_when_primary_empty(monkeypatch):
monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None))
monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
deezer_client = _FakeSourceClient(artist_id="dz-artist", albums=[], image_url="https://example.com/deezer.jpg")
spotify_album = types.SimpleNamespace(id="sp-album", name="Spotify Album", release_date=None)
spotify_client = _FakeSourceClient(artist_id="sp-artist", albums=[spotify_album], image_url="https://example.com/spotify.jpg")
def fake_get_client_for_source(source):
return {"deezer": deezer_client, "spotify": spotify_client}.get(source)
monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source)
artist = _build_artist()
artist.spotify_artist_id = "sp-artist"
artist.deezer_artist_id = "dz-artist"
scanner = _build_scanner({"tracks": {"items": []}}, [artist])
scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False
scanner._get_lookback_period_setting = lambda: "30"
scanner._get_rescan_cutoff = lambda: None
result = scanner.get_artist_discography_for_watchlist(artist, None)
assert result is not None
assert result.source == "spotify"
assert result.artist_id == "sp-artist"
assert result.albums and result.albums[0].id == "sp-album"
assert deezer_client.album_calls
assert spotify_client.album_calls
def test_match_to_spotify_uses_strict_lookup():
spotify_client = _FakeSpotifyClient(
search_results=[types.SimpleNamespace(id="fallback-id", name="Artist One")]
)
scanner = WatchlistScanner(metadata_service=_FakeMetadataService(None, spotify_client=spotify_client))
original_get_client_for_source = watchlist_scanner_module.get_client_for_source
watchlist_scanner_module.get_client_for_source = lambda source: spotify_client if source == "spotify" else None
try:
result = scanner._match_to_spotify("Artist One")
finally:
watchlist_scanner_module.get_client_for_source = original_get_client_for_source
assert result is None
assert spotify_client.search_calls == [("Artist One", 5, False)]

708
web_server.py
View file

@ -39906,9 +39906,6 @@ def start_watchlist_scan():
database = get_database()
watchlist_artists = database.get_watchlist_artists(profile_id=scan_profile_id)
# Apply global overrides if enabled
_apply_watchlist_global_overrides(watchlist_artists)
if not watchlist_artists:
watchlist_scan_state['status'] = 'completed'
watchlist_scan_state['summary'] = {
@ -39943,82 +39940,10 @@ def start_watchlist_scan():
except Exception as backfill_error:
print(f"Error during {_bf_provider} ID backfilling: {backfill_error}")
# Continue with next provider
# IMAGE BACKFILL — fix watchlist artists with missing images
# Uses DB-only lookups (metadata cache + album art) — no API calls
try:
conn = database._get_connection()
cursor = conn.cursor()
cursor.execute("""
SELECT id, artist_name, spotify_artist_id, itunes_artist_id,
deezer_artist_id, discogs_artist_id
FROM watchlist_artists
WHERE profile_id = ? AND (image_url IS NULL OR image_url = '' OR image_url = 'None'
OR image_url NOT LIKE 'http%')
""", (scan_profile_id,))
imageless = cursor.fetchall()
if imageless:
print(f"Backfilling images for {len(imageless)} watchlist artists...")
filled = 0
for row in imageless:
name = row['artist_name']
nn = name.lower().strip()
img = None
# 1. Check metadata cache for artist image
cursor.execute("""
SELECT image_url FROM metadata_cache_entities
WHERE entity_type = 'artist' AND name = ? COLLATE NOCASE
AND image_url IS NOT NULL AND image_url LIKE 'http%'
LIMIT 1
""", (name,))
cr = cursor.fetchone()
if cr:
img = cr['image_url']
# 2. Deezer direct URL (no API call needed)
if not img and row['deezer_artist_id']:
img = f"https://api.deezer.com/artist/{row['deezer_artist_id']}/image?size=big"
# 3. Deezer ID from cache (artist may have a Deezer match we haven't stored on watchlist)
if not img:
cursor.execute("""
SELECT entity_id FROM metadata_cache_entities
WHERE entity_type = 'artist' AND source = 'deezer'
AND name = ? COLLATE NOCASE LIMIT 1
""", (name,))
dz = cursor.fetchone()
if dz and dz['entity_id']:
img = f"https://api.deezer.com/artist/{dz['entity_id']}/image?size=big"
# 4. Album art fallback (iTunes artists have no artist images)
if not img:
cursor.execute("""
SELECT image_url FROM metadata_cache_entities
WHERE entity_type = 'album' AND image_url LIKE 'http%'
AND artist_name = ? COLLATE NOCASE LIMIT 1
""", (name,))
alb = cursor.fetchone()
if alb:
img = alb['image_url']
if img:
aid = (row['spotify_artist_id'] or row['itunes_artist_id']
or row['deezer_artist_id'] or row['discogs_artist_id'])
if aid:
database.update_watchlist_artist_image(aid, img)
else:
# No external IDs — update by internal row ID directly
cursor.execute("""
UPDATE watchlist_artists SET image_url = ?, updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""", (img, row['id']))
conn.commit()
filled += 1
if filled:
print(f"Backfilled {filled}/{len(imageless)} watchlist artist images")
filled = scanner.backfill_watchlist_artist_images(scan_profile_id)
if filled:
print(f"Backfilled {filled} watchlist artist images")
except Exception as img_err:
print(f"Image backfill error: {img_err}")
@ -40043,247 +39968,26 @@ def start_watchlist_scan():
# Pause enrichment workers during scan to reduce API contention
_ew_state = _pause_enrichment_workers('watchlist scan')
# Dynamic delay calculation based on scan scope
lookback_period = scanner._get_lookback_period_setting()
is_full_discography = (lookback_period == 'all')
artist_count = len(watchlist_artists)
base_artist_delay = 2.0
base_album_delay = 0.5
# Scale up for full discography (way more albums per artist)
if is_full_discography:
base_artist_delay *= 2.0
base_album_delay *= 2.0
# Scale up further for large artist counts (sustained API pressure)
if artist_count > 200:
base_artist_delay *= 1.5
base_album_delay *= 1.25
elif artist_count > 100:
base_artist_delay *= 1.25
artist_delay = base_artist_delay
album_delay = base_album_delay
print(f"Scan parameters: {artist_count} artists, lookback={lookback_period}, "
f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album")
# Circuit breaker: pause scan on consecutive rate-limit failures
consecutive_failures = 0
CIRCUIT_BREAKER_THRESHOLD = 3
circuit_breaker_pause = 60 # seconds, doubles each trigger, max 600s
for i, artist in enumerate(watchlist_artists):
# Check for cancel request
if watchlist_scan_state.get('cancel_requested'):
print(f"[Manual Watchlist Scan] Cancel requested after {i}/{len(watchlist_artists)} artists")
watchlist_scan_state['status'] = 'cancelled'
watchlist_scan_state['current_phase'] = 'cancelled'
watchlist_scan_state['summary'] = {
'total_artists': i,
'successful_scans': len([r for r in scan_results if r.success]),
'new_tracks_found': sum(r.new_tracks_found for r in scan_results if r.success),
'tracks_added_to_wishlist': sum(r.tracks_added_to_wishlist for r in scan_results if r.success),
'cancelled': True
}
break
try:
# Fetch artist image using provider-aware method
artist_image_url = scanner.get_artist_image_url(artist) or ''
# Update progress
watchlist_scan_state.update({
'current_artist_index': i + 1,
'current_artist_name': artist.artist_name,
'current_artist_image_url': artist_image_url,
'current_phase': 'fetching_discography',
'albums_to_check': 0,
'albums_checked': 0,
'current_album': '',
'current_album_image_url': '',
'current_track_name': ''
})
# Get artist discography using provider-aware method
albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp)
if albums is None:
scan_results.append(type('ScanResult', (), {
'artist_name': artist.artist_name,
'spotify_artist_id': artist.spotify_artist_id,
'albums_checked': 0,
'new_tracks_found': 0,
'tracks_added_to_wishlist': 0,
'success': False,
'error_message': "Failed to get artist discography"
})())
continue
# Update with album count
watchlist_scan_state.update({
'current_phase': 'checking_albums',
'albums_to_check': len(albums),
'albums_checked': 0
})
# Track progress for this artist
artist_new_tracks = 0
artist_added_tracks = 0
# Scan each album
for album_index, album in enumerate(albums):
try:
# Get album tracks using provider-aware method
album_data = scanner.metadata_service.get_album(album.id)
if not album_data or 'tracks' not in album_data:
logger.debug(f"Skipping album {album.name} (id={album.id}): no track data returned")
continue
tracks = album_data['tracks']['items']
# Check release type filter (album/EP/single)
if not scanner._should_include_release(len(tracks), artist):
continue
# Get album image
album_image_url = ''
if 'images' in album_data and album_data['images']:
album_image_url = album_data['images'][0]['url']
watchlist_scan_state.update({
'albums_checked': album_index + 1,
'current_album': album.name,
'current_album_image_url': album_image_url,
'current_phase': f'checking_album_{album_index + 1}_of_{len(albums)}'
})
# Check each track
for track in tracks:
# Check content type filter (live/remix/acoustic/compilation)
if not scanner._should_include_track(track, album_data, artist):
continue
# Update current track being processed
track_name = track.get('name', 'Unknown Track')
watchlist_scan_state['current_track_name'] = track_name
if scanner.is_track_missing_from_library(track):
artist_new_tracks += 1
watchlist_scan_state['tracks_found_this_scan'] += 1
# Add to wishlist
if scanner.add_track_to_wishlist(track, album_data, artist):
artist_added_tracks += 1
watchlist_scan_state['tracks_added_this_scan'] += 1
# Add to recent wishlist additions feed
track_artists = track.get('artists', [])
track_artist_name = track_artists[0].get('name', 'Unknown Artist') if track_artists else 'Unknown Artist'
watchlist_scan_state['recent_wishlist_additions'].insert(0, {
'track_name': track_name,
'artist_name': track_artist_name,
'album_image_url': album_image_url
})
# Keep only last 10
if len(watchlist_scan_state['recent_wishlist_additions']) > 10:
watchlist_scan_state['recent_wishlist_additions'].pop()
# Rate-limited delay between albums
import time
time.sleep(album_delay)
except Exception as e:
print(f"Error checking album {album.name}: {e}")
continue
# Update scan timestamp
scanner.update_artist_scan_timestamp(artist)
# Store result
scan_results.append(type('ScanResult', (), {
'artist_name': artist.artist_name,
'spotify_artist_id': artist.spotify_artist_id,
'albums_checked': len(albums),
'new_tracks_found': artist_new_tracks,
'tracks_added_to_wishlist': artist_added_tracks,
'success': True,
'error_message': None
})())
print(f"Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist")
# Fetch similar artists for discovery feature
# This is critical for the discover page to work
try:
watchlist_scan_state['current_phase'] = 'fetching_similar_artists'
source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id)
# If Spotify is authenticated, also require Spotify IDs to be present
spotify_authenticated = spotify_client and spotify_client.is_spotify_authenticated()
if database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=scan_profile_id):
print(f" Similar artists for {artist.artist_name} are cached and fresh")
# Still backfill missing iTunes IDs
scanner._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=scan_profile_id)
else:
print(f" Fetching similar artists for {artist.artist_name}...")
scanner.update_similar_artists(artist, profile_id=scan_profile_id)
print(f" Similar artists updated for {artist.artist_name}")
except Exception as similar_error:
print(f" Failed to update similar artists for {artist.artist_name}: {similar_error}")
# Delay between artists
if i < len(watchlist_artists) - 1:
watchlist_scan_state['current_phase'] = 'rate_limiting'
time.sleep(artist_delay)
# Reset circuit breaker on successful artist scan
consecutive_failures = 0
circuit_breaker_pause = 60
except Exception as e:
print(f"Error scanning artist {artist.artist_name}: {e}")
# Circuit breaker: detect consecutive rate-limit failures
error_str = str(e).lower()
if "429" in error_str or "rate limit" in error_str:
consecutive_failures += 1
if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD:
print(f"Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s")
watchlist_scan_state['current_phase'] = 'circuit_breaker_pause'
time.sleep(circuit_breaker_pause)
circuit_breaker_pause = min(circuit_breaker_pause * 2, 600)
consecutive_failures = 0
else:
consecutive_failures = 0
scan_results.append(type('ScanResult', (), {
'artist_name': artist.artist_name,
'spotify_artist_id': artist.spotify_artist_id,
'albums_checked': 0,
'new_tracks_found': 0,
'tracks_added_to_wishlist': 0,
'success': False,
'error_message': str(e)
})())
scan_results = scanner.scan_watchlist_profile(
scan_profile_id,
watchlist_artists=watchlist_artists,
scan_state=watchlist_scan_state,
cancel_check=lambda: watchlist_scan_state.get('cancel_requested', False),
)
# Store final results (skip if cancelled — already set by cancel handler)
was_cancelled = watchlist_scan_state.get('cancel_requested', False)
if not was_cancelled:
watchlist_scan_state['status'] = 'completed'
watchlist_scan_state['results'] = scan_results
watchlist_scan_state['completed_at'] = datetime.now()
_artmap_cache_invalidate(scan_profile_id)
watchlist_scan_state['current_phase'] = 'completed'
# Calculate summary
successful_scans = [r for r in scan_results if r.success]
total_new_tracks = sum(r.new_tracks_found for r in successful_scans)
total_added_to_wishlist = sum(r.tracks_added_to_wishlist for r in successful_scans)
watchlist_scan_state['status'] = 'completed'
watchlist_scan_state['results'] = scan_results
watchlist_scan_state['completed_at'] = datetime.now()
watchlist_scan_state['current_phase'] = 'completed'
watchlist_scan_state['summary'] = {
'total_artists': len(scan_results),
'successful_scans': len(successful_scans),
@ -40361,6 +40065,15 @@ def start_watchlist_scan():
import traceback
traceback.print_exc()
# Generate Last.fm radio playlists (weekly refresh)
print("Starting Last.fm radio generation...")
watchlist_scan_state['current_phase'] = 'generating_lastfm_radio'
try:
scanner._generate_lastfm_radio_playlists()
print("Last.fm radio generation complete")
except Exception as lastfm_error:
print(f"Error generating Last.fm radio playlists: {lastfm_error}")
# Sync Spotify library cache
print("Syncing Spotify library cache...")
watchlist_scan_state['current_phase'] = 'syncing_spotify_library'
@ -40877,33 +40590,6 @@ def watchlist_global_config():
traceback.print_exc()
return jsonify({"success": False, "error": str(e)}), 500
def _apply_watchlist_global_overrides(watchlist_artists):
"""If global override is enabled, overwrite per-artist settings on WatchlistArtist objects."""
if not config_manager.get('watchlist.global_override_enabled', False):
return
# Read global settings once
g_albums = config_manager.get('watchlist.global_include_albums', True)
g_eps = config_manager.get('watchlist.global_include_eps', True)
g_singles = config_manager.get('watchlist.global_include_singles', True)
g_live = config_manager.get('watchlist.global_include_live', False)
g_remixes = config_manager.get('watchlist.global_include_remixes', False)
g_acoustic = config_manager.get('watchlist.global_include_acoustic', False)
g_compilations = config_manager.get('watchlist.global_include_compilations', False)
g_instrumentals = config_manager.get('watchlist.global_include_instrumentals', False)
print(f"[Watchlist] Global override is ACTIVE — applying to {len(watchlist_artists)} artists "
f"(albums={g_albums}, eps={g_eps}, singles={g_singles}, live={g_live}, "
f"remixes={g_remixes}, acoustic={g_acoustic}, compilations={g_compilations}, "
f"instrumentals={g_instrumentals})")
for artist in watchlist_artists:
artist.include_albums = g_albums
artist.include_eps = g_eps
artist.include_singles = g_singles
artist.include_live = g_live
artist.include_remixes = g_remixes
artist.include_acoustic = g_acoustic
artist.include_compilations = g_compilations
artist.include_instrumentals = g_instrumentals
def _update_similar_artists_worker():
"""Background worker to update similar artists for all watchlist artists"""
global similar_artists_update_state
@ -41058,8 +40744,13 @@ def _process_watchlist_scan_automatically(automation_id=None, profile_id=None):
scanner = get_watchlist_scanner(spotify_client)
all_profiles = scan_profiles # Used later for discovery pool population
# Apply global overrides if enabled
_apply_watchlist_global_overrides(watchlist_artists)
for p in scan_profiles:
try:
filled = scanner.backfill_watchlist_artist_images(p['id'])
if filled:
print(f"Backfilled {filled} watchlist artist images for profile {p['id']}")
except Exception as img_err:
print(f"Image backfill error for profile {p['id']}: {img_err}")
# Initialize detailed progress tracking (same as manual scan)
watchlist_scan_state = {
@ -41089,259 +40780,79 @@ def _process_watchlist_scan_automatically(automation_id=None, profile_id=None):
# Pause enrichment workers during scan to reduce API contention
_ew_state = _pause_enrichment_workers('auto-watchlist scan')
# Dynamic delay calculation based on scan scope
lookback_period = scanner._get_lookback_period_setting()
is_full_discography = (lookback_period == 'all')
artist_count = len(watchlist_artists)
base_artist_delay = 2.0
base_album_delay = 0.5
# Scale up for full discography (way more albums per artist)
if is_full_discography:
base_artist_delay *= 2.0
base_album_delay *= 2.0
# Scale up further for large artist counts (sustained API pressure)
if artist_count > 200:
base_artist_delay *= 1.5
base_album_delay *= 1.25
elif artist_count > 100:
base_artist_delay *= 1.25
artist_delay = base_artist_delay
album_delay = base_album_delay
print(f"[Auto-Watchlist] Scan parameters: {artist_count} artists, lookback={lookback_period}, "
f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album")
# Circuit breaker: pause scan on consecutive rate-limit failures
consecutive_failures = 0
CIRCUIT_BREAKER_THRESHOLD = 3
circuit_breaker_pause = 60 # seconds, doubles each trigger, max 600s
# Scan each artist with detailed tracking
for i, artist in enumerate(watchlist_artists):
# Check for cancel request
if watchlist_scan_state.get('cancel_requested'):
print(f"[Auto-Watchlist] Cancel requested after {i}/{len(watchlist_artists)} artists")
watchlist_scan_state['status'] = 'cancelled'
watchlist_scan_state['current_phase'] = 'cancelled'
watchlist_scan_state['summary'] = {
'total_artists': i,
'successful_scans': len([r for r in scan_results if r.success]),
'new_tracks_found': sum(r.new_tracks_found for r in scan_results if r.success),
'tracks_added_to_wishlist': sum(r.tracks_added_to_wishlist for r in scan_results if r.success),
'cancelled': True
}
_update_automation_progress(automation_id, progress=100, phase='Cancelled by user',
log_line='Scan cancelled by user', log_type='warning')
break
try:
# Fetch artist image using provider-aware method
artist_image_url = scanner.get_artist_image_url(artist) or ''
pct = 5 + (i / max(1, len(watchlist_artists))) * 90
_update_automation_progress(automation_id, progress=pct,
phase=f'Scanning: {artist.artist_name} ({i+1}/{len(watchlist_artists)})',
current_item=artist.artist_name,
processed=i, total=len(watchlist_artists))
# Update progress
watchlist_scan_state.update({
'current_artist_index': i + 1,
'current_artist_name': artist.artist_name,
'current_artist_image_url': artist_image_url,
'current_phase': 'fetching_discography',
'albums_to_check': 0,
'albums_checked': 0,
'current_album': '',
'current_album_image_url': '',
'current_track_name': ''
})
# Get artist discography using provider-aware method
albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp)
if albums is None:
scan_results.append(type('ScanResult', (), {
'artist_name': artist.artist_name,
'spotify_artist_id': artist.spotify_artist_id,
'albums_checked': 0,
'new_tracks_found': 0,
'tracks_added_to_wishlist': 0,
'success': False,
'error_message': "Failed to get artist discography"
})())
continue
# Update with album count
watchlist_scan_state.update({
'current_phase': 'checking_albums',
'albums_to_check': len(albums),
'albums_checked': 0
})
# Track progress for this artist
artist_new_tracks = 0
artist_added_tracks = 0
# Scan each album
for album_index, album in enumerate(albums):
try:
# Get album tracks using provider-aware method
album_data = scanner.metadata_service.get_album(album.id)
if not album_data or 'tracks' not in album_data:
logger.debug(f"Skipping album {album.name} (id={album.id}): no track data returned")
continue
tracks = album_data['tracks']['items']
# Check release type filter (album/EP/single)
if not scanner._should_include_release(len(tracks), artist):
continue
# Get album image
album_image_url = ''
if 'images' in album_data and album_data['images']:
album_image_url = album_data['images'][0]['url']
watchlist_scan_state.update({
'albums_checked': album_index + 1,
'current_album': album.name,
'current_album_image_url': album_image_url,
'current_phase': f'checking_album_{album_index + 1}_of_{len(albums)}'
})
# Check each track
for track in tracks:
# Check content type filter (live/remix/acoustic/compilation)
if not scanner._should_include_track(track, album_data, artist):
continue
# Update current track being processed
track_name = track.get('name', 'Unknown Track')
watchlist_scan_state['current_track_name'] = track_name
if scanner.is_track_missing_from_library(track):
artist_new_tracks += 1
watchlist_scan_state['tracks_found_this_scan'] += 1
# Add to wishlist
if scanner.add_track_to_wishlist(track, album_data, artist):
artist_added_tracks += 1
watchlist_scan_state['tracks_added_this_scan'] += 1
# Add to recent wishlist additions feed
track_artists = track.get('artists', [])
track_artist_name = track_artists[0].get('name', 'Unknown Artist') if track_artists else 'Unknown Artist'
watchlist_scan_state['recent_wishlist_additions'].insert(0, {
'track_name': track_name,
'artist_name': track_artist_name,
'album_image_url': album_image_url
})
# Keep only last 10
if len(watchlist_scan_state['recent_wishlist_additions']) > 10:
watchlist_scan_state['recent_wishlist_additions'].pop()
# Rate-limited delay between albums
import time
time.sleep(album_delay)
except Exception as e:
print(f"Error checking album {album.name}: {e}")
continue
# Update scan timestamp
scanner.update_artist_scan_timestamp(artist)
# Store result
scan_results.append(type('ScanResult', (), {
'artist_name': artist.artist_name,
'spotify_artist_id': artist.spotify_artist_id,
'albums_checked': len(albums),
'new_tracks_found': artist_new_tracks,
'tracks_added_to_wishlist': artist_added_tracks,
'success': True,
'error_message': None
})())
print(f"Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist")
if artist_new_tracks > 0:
_update_automation_progress(automation_id,
log_line=f'{artist.artist_name} — {artist_new_tracks} new, {artist_added_tracks} added', log_type='success')
def _scan_progress(event_type, payload):
if event_type == 'scan_started':
_update_automation_progress(
automation_id,
progress=5,
phase='Loading watchlist',
log_line=f"{len(watchlist_artists)} artists ({profile_label})",
log_type='info',
)
elif event_type == 'artist_started':
total = max(1, payload.get('total_artists', len(watchlist_artists)))
idx = payload.get('artist_index', 1)
artist_name = payload.get('artist_name', '')
pct = 5 + ((idx - 1) / total) * 90
_update_automation_progress(
automation_id,
progress=pct,
phase=f'Scanning: {artist_name} ({idx}/{total})',
current_item=artist_name,
processed=idx - 1,
total=total,
)
elif event_type == 'artist_completed':
artist_name = payload.get('artist_name', '')
new_tracks = payload.get('new_tracks_found', 0)
added = payload.get('tracks_added_to_wishlist', 0)
if new_tracks > 0:
_update_automation_progress(
automation_id,
log_line=f'{artist_name} — {new_tracks} new, {added} added',
log_type='success',
)
else:
_update_automation_progress(automation_id,
log_line=f'{artist.artist_name} — no new tracks', log_type='skip')
_update_automation_progress(
automation_id,
log_line=f'{artist_name} — no new tracks',
log_type='skip',
)
elif event_type == 'artist_error':
artist_name = payload.get('artist_name', '')
error_message = payload.get('error_message', 'error')
_update_automation_progress(
automation_id,
log_line=f'{artist_name} — error: {error_message[:60]}',
log_type='error',
)
elif event_type == 'cancelled':
_update_automation_progress(
automation_id,
progress=100,
phase='Cancelled by user',
log_line='Scan cancelled by user',
log_type='warning',
)
elif event_type == 'scan_completed':
_update_automation_progress(
automation_id,
progress=95,
phase='Scan complete',
log_line=(
f"Scanned {payload.get('successful_scans', 0)} artists — "
f"{payload.get('new_tracks_found', 0)} new tracks, "
f"{payload.get('tracks_added_to_wishlist', 0)} added to wishlist"
),
log_type='success' if payload.get('new_tracks_found', 0) > 0 else 'info',
)
# Emit watchlist_new_release event if new tracks were found
if artist_new_tracks > 0:
try:
if automation_engine:
automation_engine.emit('watchlist_new_release', {
'artist': artist.artist_name,
'new_tracks': str(artist_new_tracks),
'added_to_wishlist': str(artist_added_tracks),
})
except Exception:
pass
# Fetch similar artists for discovery feature (per-profile)
try:
watchlist_scan_state['current_phase'] = 'fetching_similar_artists'
source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id)
artist_profile_id = getattr(artist, 'profile_id', 1)
spotify_authenticated = spotify_client and spotify_client.is_spotify_authenticated()
if database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=artist_profile_id):
print(f" Similar artists for {artist.artist_name} are cached and fresh (profile {artist_profile_id})")
scanner._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=artist_profile_id)
else:
print(f" Fetching similar artists for {artist.artist_name} (profile {artist_profile_id})...")
scanner.update_similar_artists(artist, profile_id=artist_profile_id)
print(f" Similar artists updated for {artist.artist_name}")
except Exception as similar_error:
print(f" Failed to update similar artists for {artist.artist_name}: {similar_error}")
# Delay between artists
if i < len(watchlist_artists) - 1:
watchlist_scan_state['current_phase'] = 'rate_limiting'
time.sleep(artist_delay)
# Reset circuit breaker on successful artist scan
consecutive_failures = 0
circuit_breaker_pause = 60
except Exception as e:
print(f"Error scanning artist {artist.artist_name}: {e}")
_update_automation_progress(automation_id,
log_line=f'{artist.artist_name} — error: {str(e)[:60]}', log_type='error')
# Circuit breaker: detect consecutive rate-limit failures
error_str = str(e).lower()
if "429" in error_str or "rate limit" in error_str:
consecutive_failures += 1
if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD:
print(f"[Auto-Watchlist] Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s")
watchlist_scan_state['current_phase'] = 'circuit_breaker_pause'
time.sleep(circuit_breaker_pause)
circuit_breaker_pause = min(circuit_breaker_pause * 2, 600)
consecutive_failures = 0
else:
consecutive_failures = 0
scan_results.append(type('ScanResult', (), {
'artist_name': artist.artist_name,
'spotify_artist_id': artist.spotify_artist_id,
'albums_checked': 0,
'new_tracks_found': 0,
'tracks_added_to_wishlist': 0,
'success': False,
'error_message': str(e)
})())
continue
scan_results = scanner.scan_watchlist_artists(
watchlist_artists,
scan_state=watchlist_scan_state,
progress_callback=_scan_progress,
cancel_check=lambda: watchlist_scan_state.get('cancel_requested'),
)
# Update state with results (skip if cancelled — already set by cancel handler)
was_cancelled = watchlist_scan_state.get('cancel_requested', False)
@ -41477,6 +40988,21 @@ def _process_watchlist_scan_automatically(automation_id=None, profile_id=None):
_update_automation_progress(automation_id,
log_line=f'Seasonal error: {seasonal_error}', log_type='error')
# Generate Last.fm radio playlists (weekly refresh)
print("Starting Last.fm radio generation...")
watchlist_scan_state['current_phase'] = 'generating_lastfm_radio'
_update_automation_progress(automation_id, progress=99, phase='Generating Last.fm radio',
log_line='Building Last.fm radio playlists...', log_type='info')
try:
scanner._generate_lastfm_radio_playlists()
print("Last.fm radio generation complete")
_update_automation_progress(automation_id,
log_line='Last.fm radio playlists updated', log_type='success')
except Exception as lastfm_error:
print(f"Error generating Last.fm radio playlists: {lastfm_error}")
_update_automation_progress(automation_id,
log_line=f'Last.fm radio error: {lastfm_error}', log_type='error')
# Sync Spotify library cache
print("Syncing Spotify library cache...")
try:
@ -42001,7 +41527,7 @@ def refresh_spotify_library():
def _run_sync():
try:
from core.watchlist_scanner import get_watchlist_scanner
scanner = get_watchlist_scanner()
scanner = get_watchlist_scanner(spotify_client)
if scanner:
# Force full sync by clearing last_sync timestamp
database = get_database()