From 657d86cacebef834c65afa6f08ebe246068fc1eb Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Thu, 16 Apr 2026 08:20:48 +0300 Subject: [PATCH 01/10] Consolidate web watchlist scanning Move the shared watchlist scan loop into core/watchlist_scanner.py so web_server.py only handles triggers, locks, progress, and post-scan orchestration. Manual and scheduled watchlist scans now share the same scanner-side core, while the web entrypoints keep profile selection and automation progress updates. --- core/watchlist_scanner.py | 419 ++++++++++++++++++- tests/test_watchlist_scanner_scan.py | 199 +++++++++ web_server.py | 598 ++++----------------------- 3 files changed, 699 insertions(+), 517 deletions(-) create mode 100644 tests/test_watchlist_scanner_scan.py diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 0815a528..b20f36f2 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -4,7 +4,7 @@ Watchlist Scanner Service - Monitors watched artists for new releases """ -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Callable from datetime import datetime, timezone, timedelta from dataclasses import dataclass import re @@ -901,6 +901,423 @@ class WatchlistScanner: success=False, error_message=str(e) ) + + def _apply_global_watchlist_overrides(self, watchlist_artists: List[WatchlistArtist]): + """Apply global watchlist release-type overrides to a batch of artists.""" + try: + from config.settings import config_manager + except Exception: + return + + if not config_manager.get('watchlist.global_override_enabled', False): + return + + g_albums = config_manager.get('watchlist.global_include_albums', True) + g_eps = config_manager.get('watchlist.global_include_eps', True) + g_singles = config_manager.get('watchlist.global_include_singles', True) + g_live = config_manager.get('watchlist.global_include_live', False) + g_remixes = config_manager.get('watchlist.global_include_remixes', False) + g_acoustic = config_manager.get('watchlist.global_include_acoustic', False) + g_compilations = config_manager.get('watchlist.global_include_compilations', False) + g_instrumentals = config_manager.get('watchlist.global_include_instrumentals', False) + + logger.info( + "Applying global watchlist override to %s artists " + "(albums=%s, eps=%s, singles=%s, live=%s, remixes=%s, acoustic=%s, compilations=%s, instrumentals=%s)", + len(watchlist_artists), + g_albums, + g_eps, + g_singles, + g_live, + g_remixes, + g_acoustic, + g_compilations, + g_instrumentals, + ) + + for artist in watchlist_artists: + artist.include_albums = g_albums + artist.include_eps = g_eps + artist.include_singles = g_singles + artist.include_live = g_live + artist.include_remixes = g_remixes + artist.include_acoustic = g_acoustic + artist.include_compilations = g_compilations + artist.include_instrumentals = g_instrumentals + + def scan_watchlist_profile( + self, + profile_id: int, + watchlist_artists: Optional[List[WatchlistArtist]] = None, + *, + scan_state: Optional[Dict[str, Any]] = None, + progress_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, + cancel_check: Optional[Callable[[], bool]] = None, + artist_index_offset: int = 0, + total_artists_override: Optional[int] = None, + apply_global_overrides: bool = True, + ) -> List[ScanResult]: + """Scan a single watchlist profile using the shared watchlist scan engine.""" + if watchlist_artists is None: + watchlist_artists = self.database.get_watchlist_artists(profile_id=profile_id) + + if apply_global_overrides: + self._apply_global_watchlist_overrides(watchlist_artists) + + return self.scan_watchlist_artists( + watchlist_artists, + profile_id=profile_id, + scan_state=scan_state, + progress_callback=progress_callback, + cancel_check=cancel_check, + artist_index_offset=artist_index_offset, + total_artists_override=total_artists_override, + ) + + def scan_watchlist_artists( + self, + watchlist_artists: List[WatchlistArtist], + *, + profile_id: int = 1, + scan_state: Optional[Dict[str, Any]] = None, + progress_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, + cancel_check: Optional[Callable[[], bool]] = None, + artist_index_offset: int = 0, + total_artists_override: Optional[int] = None, + ) -> List[ScanResult]: + """Scan a list of watchlist artists using the shared web watchlist scan flow.""" + scan_results: List[ScanResult] = [] + if not watchlist_artists: + if scan_state is not None: + scan_state.update({ + 'status': 'completed', + 'total_artists': 0, + 'current_artist_index': 0, + 'current_artist_name': '', + 'current_artist_image_url': '', + 'current_phase': 'completed', + 'albums_to_check': 0, + 'albums_checked': 0, + 'current_album': '', + 'current_album_image_url': '', + 'current_track_name': '', + 'tracks_found_this_scan': 0, + 'tracks_added_this_scan': 0, + 'recent_wishlist_additions': [], + 'results': [], + 'summary': { + 'total_artists': 0, + 'successful_scans': 0, + 'new_tracks_found': 0, + 'tracks_added_to_wishlist': 0, + }, + 'completed_at': datetime.now(), + 'error': None, + }) + return scan_results + + if scan_state is not None: + scan_state.update({ + 'status': 'scanning', + 'started_at': scan_state.get('started_at') or datetime.now(), + 'total_artists': total_artists_override if total_artists_override is not None else len(watchlist_artists), + 'current_artist_index': scan_state.get('current_artist_index', artist_index_offset), + 'current_artist_name': scan_state.get('current_artist_name', ''), + 'current_artist_image_url': scan_state.get('current_artist_image_url', ''), + 'current_phase': 'starting', + 'albums_to_check': 0, + 'albums_checked': 0, + 'current_album': '', + 'current_album_image_url': '', + 'current_track_name': '', + 'tracks_found_this_scan': scan_state.get('tracks_found_this_scan', 0), + 'tracks_added_this_scan': scan_state.get('tracks_added_this_scan', 0), + 'recent_wishlist_additions': scan_state.get('recent_wishlist_additions', []), + 'results': scan_state.get('results', []), + 'summary': scan_state.get('summary', {}), + 'error': None, + }) + + def _emit(event_type: str, **payload): + if progress_callback: + try: + progress_callback(event_type, payload) + except Exception: + logger.debug("Watchlist scan progress callback failed for %s", event_type, exc_info=True) + + _emit('scan_started', profile_id=profile_id, total_artists=len(watchlist_artists)) + + # Rate-limit-aware backfill of artist IDs for the providers we can safely resolve. + providers_to_backfill = ['itunes', 'deezer'] + if self.spotify_client and self.spotify_client.is_spotify_authenticated(): + providers_to_backfill.append('spotify') + try: + from config.settings import config_manager as _cfg + if _cfg.get('discogs.token', ''): + providers_to_backfill.append('discogs') + except Exception: + pass + + for provider in providers_to_backfill: + try: + logger.info("Checking for missing %s IDs in watchlist...", provider) + self._backfill_missing_ids(watchlist_artists, provider) + except Exception as backfill_error: + logger.warning("Error during %s ID backfilling: %s", provider, backfill_error) + + lookback_period = self._get_lookback_period_setting() + is_full_discography = (lookback_period == 'all') + artist_count = len(watchlist_artists) + + base_artist_delay = DELAY_BETWEEN_ARTISTS + base_album_delay = DELAY_BETWEEN_ALBUMS + if is_full_discography: + base_artist_delay *= 2.0 + base_album_delay *= 2.0 + if artist_count > 200: + base_artist_delay *= 1.5 + base_album_delay *= 1.25 + elif artist_count > 100: + base_artist_delay *= 1.25 + + artist_delay = base_artist_delay + album_delay = base_album_delay + logger.info( + "Scan parameters: %s artists, lookback=%s, delays: %.1fs/artist, %.1fs/album", + artist_count, + lookback_period, + artist_delay, + album_delay, + ) + + for i, artist in enumerate(watchlist_artists): + if cancel_check and cancel_check(): + logger.info("Watchlist scan cancelled after %s/%s artists", i, len(watchlist_artists)) + if scan_state is not None: + successful_scans = [r for r in scan_results if r.success] + scan_state['status'] = 'cancelled' + scan_state['current_phase'] = 'cancelled' + scan_state['summary'] = { + 'total_artists': i, + 'successful_scans': len(successful_scans), + 'new_tracks_found': sum(r.new_tracks_found for r in successful_scans), + 'tracks_added_to_wishlist': sum(r.tracks_added_to_wishlist for r in successful_scans), + 'cancelled': True, + } + _emit('cancelled', processed=i, total=len(watchlist_artists)) + break + + try: + artist_image_url = self.get_artist_image_url(artist) or '' + absolute_index = artist_index_offset + i + 1 + if scan_state is not None: + scan_state.update({ + 'current_artist_index': absolute_index, + 'current_artist_name': artist.artist_name, + 'current_artist_image_url': artist_image_url, + 'current_phase': 'fetching_discography', + 'albums_to_check': 0, + 'albums_checked': 0, + 'current_album': '', + 'current_album_image_url': '', + 'current_track_name': '', + }) + + _emit( + 'artist_started', + artist_name=artist.artist_name, + artist_index=absolute_index, + total_artists=total_artists_override if total_artists_override is not None else len(watchlist_artists), + profile_id=profile_id, + artist_image_url=artist_image_url, + ) + + albums = self.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp) + if albums is None: + scan_results.append(ScanResult( + artist_name=artist.artist_name, + spotify_artist_id=artist.spotify_artist_id or '', + albums_checked=0, + new_tracks_found=0, + tracks_added_to_wishlist=0, + success=False, + error_message="Failed to get artist discography", + )) + _emit( + 'artist_error', + artist_name=artist.artist_name, + profile_id=profile_id, + error_message="Failed to get artist discography", + ) + continue + + if scan_state is not None: + scan_state.update({ + 'current_phase': 'checking_albums', + 'albums_to_check': len(albums), + 'albums_checked': 0, + }) + + artist_new_tracks = 0 + artist_added_tracks = 0 + + for album_index, album in enumerate(albums): + try: + album_data = self.metadata_service.get_album(album.id) + if not album_data or 'tracks' not in album_data: + logger.debug("Skipping album %s (id=%s): no track data returned", album.name, album.id) + continue + + tracks = album_data['tracks']['items'] + if not self._should_include_release(len(tracks), artist): + continue + + album_image_url = '' + if 'images' in album_data and album_data['images']: + album_image_url = album_data['images'][0]['url'] + + if scan_state is not None: + scan_state.update({ + 'albums_checked': album_index + 1, + 'current_album': album.name, + 'current_album_image_url': album_image_url, + 'current_phase': f'checking_album_{album_index + 1}_of_{len(albums)}', + }) + + _emit( + 'album_started', + artist_name=artist.artist_name, + album_name=album.name, + album_index=album_index + 1, + total_albums=len(albums), + album_image_url=album_image_url, + ) + + for track in tracks: + if not self._should_include_track(track, album_data, artist): + continue + + track_name = track.get('name', 'Unknown Track') + if scan_state is not None: + scan_state['current_track_name'] = track_name + + if self.is_track_missing_from_library(track, album_name=album_data.get('name')): + artist_new_tracks += 1 + if scan_state is not None: + scan_state['tracks_found_this_scan'] += 1 + + if self.add_track_to_wishlist(track, album_data, artist): + artist_added_tracks += 1 + if scan_state is not None: + scan_state['tracks_added_this_scan'] += 1 + + track_artists = track.get('artists', []) + track_artist_name = track_artists[0].get('name', 'Unknown Artist') if track_artists else 'Unknown Artist' + if scan_state is not None: + scan_state['recent_wishlist_additions'].insert(0, { + 'track_name': track_name, + 'artist_name': track_artist_name, + 'album_image_url': album_image_url, + }) + if len(scan_state['recent_wishlist_additions']) > 10: + scan_state['recent_wishlist_additions'].pop() + + if album_index < len(albums) - 1: + time.sleep(album_delay) + + except Exception as e: + logger.warning("Error checking album %s: %s", album.name, e) + continue + + self.update_artist_scan_timestamp(artist) + + scan_results.append(ScanResult( + artist_name=artist.artist_name, + spotify_artist_id=artist.spotify_artist_id or '', + albums_checked=len(albums), + new_tracks_found=artist_new_tracks, + tracks_added_to_wishlist=artist_added_tracks, + success=True, + )) + + _emit( + 'artist_completed', + artist_name=artist.artist_name, + artist_index=absolute_index, + total_artists=total_artists_override if total_artists_override is not None else len(watchlist_artists), + profile_id=profile_id, + albums_checked=len(albums), + new_tracks_found=artist_new_tracks, + tracks_added_to_wishlist=artist_added_tracks, + ) + + try: + if scan_state is not None: + scan_state['current_phase'] = 'fetching_similar_artists' + source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id) + artist_profile_id = getattr(artist, 'profile_id', profile_id) + spotify_authenticated = self.spotify_client and self.spotify_client.is_spotify_authenticated() + if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=artist_profile_id): + logger.info("Similar artists for %s are cached and fresh (profile %s)", artist.artist_name, artist_profile_id) + self._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=artist_profile_id) + else: + logger.info("Fetching similar artists for %s (profile %s)...", artist.artist_name, artist_profile_id) + self.update_similar_artists(artist, profile_id=artist_profile_id) + logger.info("Similar artists updated for %s", artist.artist_name) + except Exception as similar_error: + logger.warning("Failed to update similar artists for %s: %s", artist.artist_name, similar_error) + + if i < len(watchlist_artists) - 1: + if scan_state is not None: + scan_state['current_phase'] = 'rate_limiting' + time.sleep(artist_delay) + + except Exception as e: + logger.error("Error scanning artist %s: %s", artist.artist_name, e) + scan_results.append(ScanResult( + artist_name=artist.artist_name, + spotify_artist_id=artist.spotify_artist_id or '', + albums_checked=0, + new_tracks_found=0, + tracks_added_to_wishlist=0, + success=False, + error_message=str(e), + )) + _emit( + 'artist_error', + artist_name=artist.artist_name, + artist_index=artist_index_offset + i + 1, + total_artists=total_artists_override if total_artists_override is not None else len(watchlist_artists), + profile_id=profile_id, + error_message=str(e), + ) + + if scan_state is not None: + successful_scans = [r for r in scan_results if r.success] + total_new_tracks = sum(r.new_tracks_found for r in successful_scans) + total_added_to_wishlist = sum(r.tracks_added_to_wishlist for r in successful_scans) + scan_state['results'] = list(scan_state.get('results', [])) + scan_results + if scan_state.get('status') != 'cancelled': + scan_state['status'] = 'completed' + scan_state['completed_at'] = datetime.now() + scan_state['current_phase'] = 'completed' + scan_state['summary'] = { + 'total_artists': len(scan_results), + 'successful_scans': len(successful_scans), + 'new_tracks_found': total_new_tracks, + 'tracks_added_to_wishlist': total_added_to_wishlist, + } + + _emit( + 'scan_completed', + profile_id=profile_id, + total_artists=len(watchlist_artists), + total_scanned=len(scan_results), + successful_scans=len([r for r in scan_results if r.success]), + new_tracks_found=sum(r.new_tracks_found for r in scan_results if r.success), + tracks_added_to_wishlist=sum(r.tracks_added_to_wishlist for r in scan_results if r.success), + ) + return scan_results def get_artist_discography(self, spotify_artist_id: str, last_scan_timestamp: Optional[datetime] = None) -> Optional[List]: """ diff --git a/tests/test_watchlist_scanner_scan.py b/tests/test_watchlist_scanner_scan.py new file mode 100644 index 00000000..bb59d786 --- /dev/null +++ b/tests/test_watchlist_scanner_scan.py @@ -0,0 +1,199 @@ +import sys +import types + + +if "spotipy" not in sys.modules: + spotipy = types.ModuleType("spotipy") + + class _DummySpotify: + def __init__(self, *args, **kwargs): + pass + + oauth2 = types.ModuleType("spotipy.oauth2") + + class _DummyOAuth: + def __init__(self, *args, **kwargs): + pass + + spotipy.Spotify = _DummySpotify + oauth2.SpotifyOAuth = _DummyOAuth + oauth2.SpotifyClientCredentials = _DummyOAuth + spotipy.oauth2 = oauth2 + sys.modules["spotipy"] = spotipy + sys.modules["spotipy.oauth2"] = oauth2 + +import core.watchlist_scanner as watchlist_scanner_module +from core.watchlist_scanner import WatchlistScanner + + +class _FakeSpotifyClient: + def is_spotify_authenticated(self): + return False + + +class _FakeMetadataService: + def __init__(self, album_data): + self.spotify = _FakeSpotifyClient() + self.itunes = types.SimpleNamespace() + self._album_data = album_data + + def get_album(self, album_id): + return self._album_data + + +class _FakeDB: + def __init__(self, artists): + self.artists = artists + self.similar_calls = [] + + def get_watchlist_artists(self, profile_id=None): + return list(self.artists) + + def has_fresh_similar_artists(self, *args, **kwargs): + self.similar_calls.append((args, kwargs)) + return False + + +def _build_artist(name="Artist One", profile_id=11): + return types.SimpleNamespace( + artist_name=name, + spotify_artist_id="sp-artist", + itunes_artist_id="it-artist", + deezer_artist_id="dz-artist", + discogs_artist_id="dg-artist", + last_scan_timestamp=None, + id=123, + profile_id=profile_id, + include_albums=True, + include_eps=True, + include_singles=True, + include_live=False, + include_remixes=False, + include_acoustic=False, + include_compilations=False, + include_instrumentals=False, + lookback_days=7, + image_url=None, + ) + + +def _build_scanner(album_data, artists): + scanner = WatchlistScanner(metadata_service=_FakeMetadataService(album_data)) + scanner._database = _FakeDB(artists) + scanner._wishlist_service = types.SimpleNamespace() + scanner._matching_engine = types.SimpleNamespace() + return scanner + + +def test_scan_watchlist_profile_loads_artists_and_applies_overrides(monkeypatch): + artist = _build_artist() + scanner = _build_scanner({"tracks": {"items": []}}, [artist]) + + loaded_profiles = [] + override_calls = [] + scan_calls = [] + + monkeypatch.setattr(scanner.database, "get_watchlist_artists", lambda profile_id=None: loaded_profiles.append(profile_id) or [artist]) + monkeypatch.setattr(scanner, "_apply_global_watchlist_overrides", lambda artists: override_calls.append(list(artists))) + monkeypatch.setattr(scanner, "scan_watchlist_artists", lambda artists, **kwargs: scan_calls.append((list(artists), kwargs)) or ["ok"]) + + result = scanner.scan_watchlist_profile(42) + + assert result == ["ok"] + assert loaded_profiles == [42] + assert override_calls and override_calls[0][0].artist_name == "Artist One" + assert scan_calls and scan_calls[0][0][0].artist_name == "Artist One" + assert scan_calls[0][1]["profile_id"] == 42 + + +def test_scan_watchlist_artists_scans_tracks_and_updates_state(monkeypatch): + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0) + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0) + + artist = _build_artist() + album = types.SimpleNamespace(id="album-1", name="Album One") + album_data = { + "name": "Album One", + "images": [{"url": "https://example.com/album.jpg"}], + "tracks": { + "items": [ + { + "id": "track-1", + "name": "Track One", + "track_number": 1, + "disc_number": 1, + "artists": [{"name": "Artist One"}], + } + ] + }, + } + scanner = _build_scanner(album_data, [artist]) + scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False + + monkeypatch.setattr(scanner, "_backfill_missing_ids", lambda *args, **kwargs: None) + monkeypatch.setattr(scanner, "get_artist_image_url", lambda *_args, **_kwargs: "https://example.com/artist.jpg") + monkeypatch.setattr(scanner, "get_artist_discography_for_watchlist", lambda *_args, **_kwargs: [album]) + monkeypatch.setattr(scanner, "_get_lookback_period_setting", lambda: "30") + monkeypatch.setattr(scanner, "_get_rescan_cutoff", lambda: None) + monkeypatch.setattr(scanner, "_should_include_release", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_should_include_track", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "is_track_missing_from_library", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0) + + scan_state = {} + results = scanner.scan_watchlist_artists([artist], scan_state=scan_state) + + assert len(results) == 1 + assert results[0].success is True + assert results[0].new_tracks_found == 1 + assert results[0].tracks_added_to_wishlist == 1 + assert scan_state["status"] == "completed" + assert scan_state["summary"]["successful_scans"] == 1 + assert scan_state["summary"]["new_tracks_found"] == 1 + assert scan_state["summary"]["tracks_added_to_wishlist"] == 1 + assert scan_state["recent_wishlist_additions"][0]["track_name"] == "Track One" + + +def test_scan_watchlist_artists_honors_cancel_check(monkeypatch): + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0) + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0) + + artist_a = _build_artist("Artist One") + artist_b = _build_artist("Artist Two") + album = types.SimpleNamespace(id="album-1", name="Album One") + album_data = { + "name": "Album One", + "tracks": {"items": []}, + } + scanner = _build_scanner(album_data, [artist_a, artist_b]) + scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False + + monkeypatch.setattr(scanner, "_backfill_missing_ids", lambda *args, **kwargs: None) + monkeypatch.setattr(scanner, "get_artist_image_url", lambda *_args, **_kwargs: "https://example.com/artist.jpg") + monkeypatch.setattr(scanner, "get_artist_discography_for_watchlist", lambda *_args, **_kwargs: [album]) + monkeypatch.setattr(scanner, "_get_lookback_period_setting", lambda: "30") + monkeypatch.setattr(scanner, "_get_rescan_cutoff", lambda: None) + monkeypatch.setattr(scanner, "_should_include_release", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_should_include_track", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "is_track_missing_from_library", lambda *_args, **_kwargs: False) + monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *_args, **_kwargs: False) + monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0) + + cancels = iter([False, True]) + scan_state = {} + results = scanner.scan_watchlist_artists( + [artist_a, artist_b], + scan_state=scan_state, + cancel_check=lambda: next(cancels), + ) + + assert len(results) == 1 + assert results[0].artist_name == "Artist One" + assert scan_state["status"] == "cancelled" + assert scan_state["summary"]["cancelled"] is True + assert scan_state["summary"]["successful_scans"] == 1 diff --git a/web_server.py b/web_server.py index 829ccc5c..f74db392 100644 --- a/web_server.py +++ b/web_server.py @@ -39802,9 +39802,6 @@ def start_watchlist_scan(): database = get_database() watchlist_artists = database.get_watchlist_artists(profile_id=scan_profile_id) - # Apply global overrides if enabled - _apply_watchlist_global_overrides(watchlist_artists) - if not watchlist_artists: watchlist_scan_state['status'] = 'completed' watchlist_scan_state['summary'] = { @@ -39939,247 +39936,26 @@ def start_watchlist_scan(): # Pause enrichment workers during scan to reduce API contention _ew_state = _pause_enrichment_workers('watchlist scan') - - # Dynamic delay calculation based on scan scope - lookback_period = scanner._get_lookback_period_setting() - is_full_discography = (lookback_period == 'all') - artist_count = len(watchlist_artists) - - base_artist_delay = 2.0 - base_album_delay = 0.5 - - # Scale up for full discography (way more albums per artist) - if is_full_discography: - base_artist_delay *= 2.0 - base_album_delay *= 2.0 - - # Scale up further for large artist counts (sustained API pressure) - if artist_count > 200: - base_artist_delay *= 1.5 - base_album_delay *= 1.25 - elif artist_count > 100: - base_artist_delay *= 1.25 - - artist_delay = base_artist_delay - album_delay = base_album_delay - print(f"Scan parameters: {artist_count} artists, lookback={lookback_period}, " - f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album") - - # Circuit breaker: pause scan on consecutive rate-limit failures - consecutive_failures = 0 - CIRCUIT_BREAKER_THRESHOLD = 3 - circuit_breaker_pause = 60 # seconds, doubles each trigger, max 600s - - for i, artist in enumerate(watchlist_artists): - # Check for cancel request - if watchlist_scan_state.get('cancel_requested'): - print(f"[Manual Watchlist Scan] Cancel requested after {i}/{len(watchlist_artists)} artists") - watchlist_scan_state['status'] = 'cancelled' - watchlist_scan_state['current_phase'] = 'cancelled' - watchlist_scan_state['summary'] = { - 'total_artists': i, - 'successful_scans': len([r for r in scan_results if r.success]), - 'new_tracks_found': sum(r.new_tracks_found for r in scan_results if r.success), - 'tracks_added_to_wishlist': sum(r.tracks_added_to_wishlist for r in scan_results if r.success), - 'cancelled': True - } - break - - try: - # Fetch artist image using provider-aware method - artist_image_url = scanner.get_artist_image_url(artist) or '' - - # Update progress - watchlist_scan_state.update({ - 'current_artist_index': i + 1, - 'current_artist_name': artist.artist_name, - 'current_artist_image_url': artist_image_url, - 'current_phase': 'fetching_discography', - 'albums_to_check': 0, - 'albums_checked': 0, - 'current_album': '', - 'current_album_image_url': '', - 'current_track_name': '' - }) - - # Get artist discography using provider-aware method - albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp) - - if albums is None: - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': 0, - 'new_tracks_found': 0, - 'tracks_added_to_wishlist': 0, - 'success': False, - 'error_message': "Failed to get artist discography" - })()) - continue - - # Update with album count - watchlist_scan_state.update({ - 'current_phase': 'checking_albums', - 'albums_to_check': len(albums), - 'albums_checked': 0 - }) - - # Track progress for this artist - artist_new_tracks = 0 - artist_added_tracks = 0 - - # Scan each album - for album_index, album in enumerate(albums): - try: - # Get album tracks using provider-aware method - album_data = scanner.metadata_service.get_album(album.id) - if not album_data or 'tracks' not in album_data: - logger.debug(f"Skipping album {album.name} (id={album.id}): no track data returned") - continue - - tracks = album_data['tracks']['items'] - - # Check release type filter (album/EP/single) - if not scanner._should_include_release(len(tracks), artist): - continue - - # Get album image - album_image_url = '' - if 'images' in album_data and album_data['images']: - album_image_url = album_data['images'][0]['url'] - - watchlist_scan_state.update({ - 'albums_checked': album_index + 1, - 'current_album': album.name, - 'current_album_image_url': album_image_url, - 'current_phase': f'checking_album_{album_index + 1}_of_{len(albums)}' - }) - - # Check each track - for track in tracks: - # Check content type filter (live/remix/acoustic/compilation) - if not scanner._should_include_track(track, album_data, artist): - continue - - # Update current track being processed - track_name = track.get('name', 'Unknown Track') - watchlist_scan_state['current_track_name'] = track_name - - if scanner.is_track_missing_from_library(track): - artist_new_tracks += 1 - watchlist_scan_state['tracks_found_this_scan'] += 1 - - # Add to wishlist - if scanner.add_track_to_wishlist(track, album_data, artist): - artist_added_tracks += 1 - watchlist_scan_state['tracks_added_this_scan'] += 1 - - # Add to recent wishlist additions feed - track_artists = track.get('artists', []) - track_artist_name = track_artists[0].get('name', 'Unknown Artist') if track_artists else 'Unknown Artist' - - watchlist_scan_state['recent_wishlist_additions'].insert(0, { - 'track_name': track_name, - 'artist_name': track_artist_name, - 'album_image_url': album_image_url - }) - - # Keep only last 10 - if len(watchlist_scan_state['recent_wishlist_additions']) > 10: - watchlist_scan_state['recent_wishlist_additions'].pop() - - # Rate-limited delay between albums - import time - time.sleep(album_delay) - - except Exception as e: - print(f"Error checking album {album.name}: {e}") - continue - - # Update scan timestamp - scanner.update_artist_scan_timestamp(artist) - - # Store result - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': len(albums), - 'new_tracks_found': artist_new_tracks, - 'tracks_added_to_wishlist': artist_added_tracks, - 'success': True, - 'error_message': None - })()) - - print(f"Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist") - - # Fetch similar artists for discovery feature - # This is critical for the discover page to work - try: - watchlist_scan_state['current_phase'] = 'fetching_similar_artists' - source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id) - - # If Spotify is authenticated, also require Spotify IDs to be present - spotify_authenticated = spotify_client and spotify_client.is_spotify_authenticated() - if database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=scan_profile_id): - print(f" Similar artists for {artist.artist_name} are cached and fresh") - # Still backfill missing iTunes IDs - scanner._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=scan_profile_id) - else: - print(f" Fetching similar artists for {artist.artist_name}...") - scanner.update_similar_artists(artist, profile_id=scan_profile_id) - print(f" Similar artists updated for {artist.artist_name}") - except Exception as similar_error: - print(f" Failed to update similar artists for {artist.artist_name}: {similar_error}") - - # Delay between artists - if i < len(watchlist_artists) - 1: - watchlist_scan_state['current_phase'] = 'rate_limiting' - time.sleep(artist_delay) - - # Reset circuit breaker on successful artist scan - consecutive_failures = 0 - circuit_breaker_pause = 60 - - except Exception as e: - print(f"Error scanning artist {artist.artist_name}: {e}") - - # Circuit breaker: detect consecutive rate-limit failures - error_str = str(e).lower() - if "429" in error_str or "rate limit" in error_str: - consecutive_failures += 1 - if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD: - print(f"Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s") - watchlist_scan_state['current_phase'] = 'circuit_breaker_pause' - time.sleep(circuit_breaker_pause) - circuit_breaker_pause = min(circuit_breaker_pause * 2, 600) - consecutive_failures = 0 - else: - consecutive_failures = 0 - - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': 0, - 'new_tracks_found': 0, - 'tracks_added_to_wishlist': 0, - 'success': False, - 'error_message': str(e) - })()) + scan_results = scanner.scan_watchlist_profile( + scan_profile_id, + watchlist_artists=watchlist_artists, + scan_state=watchlist_scan_state, + cancel_check=lambda: watchlist_scan_state.get('cancel_requested', False), + ) # Store final results (skip if cancelled — already set by cancel handler) was_cancelled = watchlist_scan_state.get('cancel_requested', False) if not was_cancelled: - watchlist_scan_state['status'] = 'completed' - watchlist_scan_state['results'] = scan_results - watchlist_scan_state['completed_at'] = datetime.now() _artmap_cache_invalidate(scan_profile_id) - watchlist_scan_state['current_phase'] = 'completed' - - # Calculate summary successful_scans = [r for r in scan_results if r.success] total_new_tracks = sum(r.new_tracks_found for r in successful_scans) total_added_to_wishlist = sum(r.tracks_added_to_wishlist for r in successful_scans) + watchlist_scan_state['status'] = 'completed' + watchlist_scan_state['results'] = scan_results + watchlist_scan_state['completed_at'] = datetime.now() + watchlist_scan_state['current_phase'] = 'completed' + watchlist_scan_state['summary'] = { 'total_artists': len(scan_results), 'successful_scans': len(successful_scans), @@ -40773,33 +40549,6 @@ def watchlist_global_config(): traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 -def _apply_watchlist_global_overrides(watchlist_artists): - """If global override is enabled, overwrite per-artist settings on WatchlistArtist objects.""" - if not config_manager.get('watchlist.global_override_enabled', False): - return - # Read global settings once - g_albums = config_manager.get('watchlist.global_include_albums', True) - g_eps = config_manager.get('watchlist.global_include_eps', True) - g_singles = config_manager.get('watchlist.global_include_singles', True) - g_live = config_manager.get('watchlist.global_include_live', False) - g_remixes = config_manager.get('watchlist.global_include_remixes', False) - g_acoustic = config_manager.get('watchlist.global_include_acoustic', False) - g_compilations = config_manager.get('watchlist.global_include_compilations', False) - g_instrumentals = config_manager.get('watchlist.global_include_instrumentals', False) - print(f"[Watchlist] Global override is ACTIVE — applying to {len(watchlist_artists)} artists " - f"(albums={g_albums}, eps={g_eps}, singles={g_singles}, live={g_live}, " - f"remixes={g_remixes}, acoustic={g_acoustic}, compilations={g_compilations}, " - f"instrumentals={g_instrumentals})") - for artist in watchlist_artists: - artist.include_albums = g_albums - artist.include_eps = g_eps - artist.include_singles = g_singles - artist.include_live = g_live - artist.include_remixes = g_remixes - artist.include_acoustic = g_acoustic - artist.include_compilations = g_compilations - artist.include_instrumentals = g_instrumentals - def _update_similar_artists_worker(): """Background worker to update similar artists for all watchlist artists""" global similar_artists_update_state @@ -40954,9 +40703,6 @@ def _process_watchlist_scan_automatically(automation_id=None, profile_id=None): scanner = get_watchlist_scanner(spotify_client) all_profiles = scan_profiles # Used later for discovery pool population - # Apply global overrides if enabled - _apply_watchlist_global_overrides(watchlist_artists) - # Initialize detailed progress tracking (same as manual scan) watchlist_scan_state = { 'status': 'scanning', @@ -40985,259 +40731,79 @@ def _process_watchlist_scan_automatically(automation_id=None, profile_id=None): # Pause enrichment workers during scan to reduce API contention _ew_state = _pause_enrichment_workers('auto-watchlist scan') - # Dynamic delay calculation based on scan scope - lookback_period = scanner._get_lookback_period_setting() - is_full_discography = (lookback_period == 'all') - artist_count = len(watchlist_artists) - - base_artist_delay = 2.0 - base_album_delay = 0.5 - - # Scale up for full discography (way more albums per artist) - if is_full_discography: - base_artist_delay *= 2.0 - base_album_delay *= 2.0 - - # Scale up further for large artist counts (sustained API pressure) - if artist_count > 200: - base_artist_delay *= 1.5 - base_album_delay *= 1.25 - elif artist_count > 100: - base_artist_delay *= 1.25 - - artist_delay = base_artist_delay - album_delay = base_album_delay - print(f"[Auto-Watchlist] Scan parameters: {artist_count} artists, lookback={lookback_period}, " - f"delays: {artist_delay:.1f}s/artist, {album_delay:.1f}s/album") - - # Circuit breaker: pause scan on consecutive rate-limit failures - consecutive_failures = 0 - CIRCUIT_BREAKER_THRESHOLD = 3 - circuit_breaker_pause = 60 # seconds, doubles each trigger, max 600s - - # Scan each artist with detailed tracking - for i, artist in enumerate(watchlist_artists): - # Check for cancel request - if watchlist_scan_state.get('cancel_requested'): - print(f"[Auto-Watchlist] Cancel requested after {i}/{len(watchlist_artists)} artists") - watchlist_scan_state['status'] = 'cancelled' - watchlist_scan_state['current_phase'] = 'cancelled' - watchlist_scan_state['summary'] = { - 'total_artists': i, - 'successful_scans': len([r for r in scan_results if r.success]), - 'new_tracks_found': sum(r.new_tracks_found for r in scan_results if r.success), - 'tracks_added_to_wishlist': sum(r.tracks_added_to_wishlist for r in scan_results if r.success), - 'cancelled': True - } - _update_automation_progress(automation_id, progress=100, phase='Cancelled by user', - log_line='Scan cancelled by user', log_type='warning') - break - - try: - # Fetch artist image using provider-aware method - artist_image_url = scanner.get_artist_image_url(artist) or '' - - pct = 5 + (i / max(1, len(watchlist_artists))) * 90 - _update_automation_progress(automation_id, progress=pct, - phase=f'Scanning: {artist.artist_name} ({i+1}/{len(watchlist_artists)})', - current_item=artist.artist_name, - processed=i, total=len(watchlist_artists)) - - # Update progress - watchlist_scan_state.update({ - 'current_artist_index': i + 1, - 'current_artist_name': artist.artist_name, - 'current_artist_image_url': artist_image_url, - 'current_phase': 'fetching_discography', - 'albums_to_check': 0, - 'albums_checked': 0, - 'current_album': '', - 'current_album_image_url': '', - 'current_track_name': '' - }) - - # Get artist discography using provider-aware method - albums = scanner.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp) - - if albums is None: - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': 0, - 'new_tracks_found': 0, - 'tracks_added_to_wishlist': 0, - 'success': False, - 'error_message': "Failed to get artist discography" - })()) - continue - - # Update with album count - watchlist_scan_state.update({ - 'current_phase': 'checking_albums', - 'albums_to_check': len(albums), - 'albums_checked': 0 - }) - - # Track progress for this artist - artist_new_tracks = 0 - artist_added_tracks = 0 - - # Scan each album - for album_index, album in enumerate(albums): - try: - # Get album tracks using provider-aware method - album_data = scanner.metadata_service.get_album(album.id) - if not album_data or 'tracks' not in album_data: - logger.debug(f"Skipping album {album.name} (id={album.id}): no track data returned") - continue - - tracks = album_data['tracks']['items'] - - # Check release type filter (album/EP/single) - if not scanner._should_include_release(len(tracks), artist): - continue - - # Get album image - album_image_url = '' - if 'images' in album_data and album_data['images']: - album_image_url = album_data['images'][0]['url'] - - watchlist_scan_state.update({ - 'albums_checked': album_index + 1, - 'current_album': album.name, - 'current_album_image_url': album_image_url, - 'current_phase': f'checking_album_{album_index + 1}_of_{len(albums)}' - }) - - # Check each track - for track in tracks: - # Check content type filter (live/remix/acoustic/compilation) - if not scanner._should_include_track(track, album_data, artist): - continue - - # Update current track being processed - track_name = track.get('name', 'Unknown Track') - watchlist_scan_state['current_track_name'] = track_name - - if scanner.is_track_missing_from_library(track): - artist_new_tracks += 1 - watchlist_scan_state['tracks_found_this_scan'] += 1 - - # Add to wishlist - if scanner.add_track_to_wishlist(track, album_data, artist): - artist_added_tracks += 1 - watchlist_scan_state['tracks_added_this_scan'] += 1 - - # Add to recent wishlist additions feed - track_artists = track.get('artists', []) - track_artist_name = track_artists[0].get('name', 'Unknown Artist') if track_artists else 'Unknown Artist' - - watchlist_scan_state['recent_wishlist_additions'].insert(0, { - 'track_name': track_name, - 'artist_name': track_artist_name, - 'album_image_url': album_image_url - }) - - # Keep only last 10 - if len(watchlist_scan_state['recent_wishlist_additions']) > 10: - watchlist_scan_state['recent_wishlist_additions'].pop() - - # Rate-limited delay between albums - import time - time.sleep(album_delay) - - except Exception as e: - print(f"Error checking album {album.name}: {e}") - continue - - # Update scan timestamp - scanner.update_artist_scan_timestamp(artist) - - # Store result - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': len(albums), - 'new_tracks_found': artist_new_tracks, - 'tracks_added_to_wishlist': artist_added_tracks, - 'success': True, - 'error_message': None - })()) - - print(f"Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist") - if artist_new_tracks > 0: - _update_automation_progress(automation_id, - log_line=f'{artist.artist_name} — {artist_new_tracks} new, {artist_added_tracks} added', log_type='success') + def _scan_progress(event_type, payload): + if event_type == 'scan_started': + _update_automation_progress( + automation_id, + progress=5, + phase='Loading watchlist', + log_line=f"{len(watchlist_artists)} artists ({profile_label})", + log_type='info', + ) + elif event_type == 'artist_started': + total = max(1, payload.get('total_artists', len(watchlist_artists))) + idx = payload.get('artist_index', 1) + artist_name = payload.get('artist_name', '') + pct = 5 + ((idx - 1) / total) * 90 + _update_automation_progress( + automation_id, + progress=pct, + phase=f'Scanning: {artist_name} ({idx}/{total})', + current_item=artist_name, + processed=idx - 1, + total=total, + ) + elif event_type == 'artist_completed': + artist_name = payload.get('artist_name', '') + new_tracks = payload.get('new_tracks_found', 0) + added = payload.get('tracks_added_to_wishlist', 0) + if new_tracks > 0: + _update_automation_progress( + automation_id, + log_line=f'{artist_name} — {new_tracks} new, {added} added', + log_type='success', + ) else: - _update_automation_progress(automation_id, - log_line=f'{artist.artist_name} — no new tracks', log_type='skip') + _update_automation_progress( + automation_id, + log_line=f'{artist_name} — no new tracks', + log_type='skip', + ) + elif event_type == 'artist_error': + artist_name = payload.get('artist_name', '') + error_message = payload.get('error_message', 'error') + _update_automation_progress( + automation_id, + log_line=f'{artist_name} — error: {error_message[:60]}', + log_type='error', + ) + elif event_type == 'cancelled': + _update_automation_progress( + automation_id, + progress=100, + phase='Cancelled by user', + log_line='Scan cancelled by user', + log_type='warning', + ) + elif event_type == 'scan_completed': + _update_automation_progress( + automation_id, + progress=95, + phase='Scan complete', + log_line=( + f"Scanned {payload.get('successful_scans', 0)} artists — " + f"{payload.get('new_tracks_found', 0)} new tracks, " + f"{payload.get('tracks_added_to_wishlist', 0)} added to wishlist" + ), + log_type='success' if payload.get('new_tracks_found', 0) > 0 else 'info', + ) - # Emit watchlist_new_release event if new tracks were found - if artist_new_tracks > 0: - try: - if automation_engine: - automation_engine.emit('watchlist_new_release', { - 'artist': artist.artist_name, - 'new_tracks': str(artist_new_tracks), - 'added_to_wishlist': str(artist_added_tracks), - }) - except Exception: - pass - - # Fetch similar artists for discovery feature (per-profile) - try: - watchlist_scan_state['current_phase'] = 'fetching_similar_artists' - source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id) - artist_profile_id = getattr(artist, 'profile_id', 1) - - spotify_authenticated = spotify_client and spotify_client.is_spotify_authenticated() - if database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=artist_profile_id): - print(f" Similar artists for {artist.artist_name} are cached and fresh (profile {artist_profile_id})") - scanner._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=artist_profile_id) - else: - print(f" Fetching similar artists for {artist.artist_name} (profile {artist_profile_id})...") - scanner.update_similar_artists(artist, profile_id=artist_profile_id) - print(f" Similar artists updated for {artist.artist_name}") - except Exception as similar_error: - print(f" Failed to update similar artists for {artist.artist_name}: {similar_error}") - - # Delay between artists - if i < len(watchlist_artists) - 1: - watchlist_scan_state['current_phase'] = 'rate_limiting' - time.sleep(artist_delay) - - # Reset circuit breaker on successful artist scan - consecutive_failures = 0 - circuit_breaker_pause = 60 - - except Exception as e: - print(f"Error scanning artist {artist.artist_name}: {e}") - _update_automation_progress(automation_id, - log_line=f'{artist.artist_name} — error: {str(e)[:60]}', log_type='error') - - # Circuit breaker: detect consecutive rate-limit failures - error_str = str(e).lower() - if "429" in error_str or "rate limit" in error_str: - consecutive_failures += 1 - if consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD: - print(f"[Auto-Watchlist] Circuit breaker: {consecutive_failures} consecutive rate-limit failures, pausing {circuit_breaker_pause}s") - watchlist_scan_state['current_phase'] = 'circuit_breaker_pause' - time.sleep(circuit_breaker_pause) - circuit_breaker_pause = min(circuit_breaker_pause * 2, 600) - consecutive_failures = 0 - else: - consecutive_failures = 0 - - scan_results.append(type('ScanResult', (), { - 'artist_name': artist.artist_name, - 'spotify_artist_id': artist.spotify_artist_id, - 'albums_checked': 0, - 'new_tracks_found': 0, - 'tracks_added_to_wishlist': 0, - 'success': False, - 'error_message': str(e) - })()) - continue + scan_results = scanner.scan_watchlist_artists( + watchlist_artists, + scan_state=watchlist_scan_state, + progress_callback=_scan_progress, + cancel_check=lambda: watchlist_scan_state.get('cancel_requested'), + ) # Update state with results (skip if cancelled — already set by cancel handler) was_cancelled = watchlist_scan_state.get('cancel_requested', False) From 40fa139804669355f542dbbb553bff60508e69c8 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Thu, 16 Apr 2026 08:27:41 +0300 Subject: [PATCH 02/10] Remove dead watchlist scan paths Drop the legacy watchlist scan entrypoints that are no longer used by the web scan flow, and keep the live refresh path pointed at the shared scanner helper. --- core/watchlist_scanner.py | 333 -------------------------------------- web_server.py | 2 +- 2 files changed, 1 insertion(+), 334 deletions(-) diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index b20f36f2..069584b3 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -398,11 +398,6 @@ class WatchlistScanner: self._metadata_service = MetadataService() return self._metadata_service - def _reset_spotify_run_state(self): - """Clear per-run Spotify suppression state.""" - self._spotify_disabled_for_run = False - self._spotify_disabled_reason = None - def _disable_spotify_for_run(self, reason: str): """Disable Spotify for rest of current run, once.""" if not self._spotify_disabled_for_run: @@ -574,334 +569,6 @@ class WatchlistScanner: return albums - def scan_all_watchlist_artists(self) -> List[ScanResult]: - """ - Scan artists in the watchlist for new releases. - - OPTIMIZED: Scans up to 50 artists per run using smart selection: - - Priority: Artists not scanned in 7+ days (guaranteed) - - Remainder: Random selection from other artists - - This reduces API calls while ensuring all artists scanned at least weekly. - Only checks releases after their last scan timestamp. - """ - logger.info("Starting watchlist scan") - - try: - self._reset_spotify_run_state() - from datetime import datetime, timedelta - import random - - # Get all watchlist artists - all_watchlist_artists = self.database.get_watchlist_artists() - if not all_watchlist_artists: - logger.info("No artists in watchlist to scan") - return [] - - logger.info(f"Found {len(all_watchlist_artists)} total artists in watchlist") - - # OPTIMIZATION: Select up to 50 artists to scan - # 1. Must scan: Artists not scanned in 7+ days (or never scanned) - seven_days_ago = datetime.now() - timedelta(days=7) - must_scan = [] - can_skip = [] - - for artist in all_watchlist_artists: - if artist.last_scan_timestamp is None: - # Never scanned - must scan - must_scan.append(artist) - elif artist.last_scan_timestamp < seven_days_ago: - # Not scanned in 7+ days - must scan - must_scan.append(artist) - else: - # Scanned recently - can skip (but might randomly select) - can_skip.append(artist) - - logger.info(f"Artists requiring scan (not scanned in 7+ days): {len(must_scan)}") - logger.info(f"Artists scanned recently (< 7 days): {len(can_skip)}") - - # 2. Fill remaining slots (up to 50 total) with random selection - max_artists_per_scan = 50 - artists_to_scan = must_scan.copy() - - remaining_slots = max_artists_per_scan - len(must_scan) - if remaining_slots > 0 and can_skip: - # Randomly sample from recently-scanned artists - random_sample_size = min(remaining_slots, len(can_skip)) - random_selection = random.sample(can_skip, random_sample_size) - artists_to_scan.extend(random_selection) - logger.info(f"Additionally scanning {len(random_selection)} randomly selected artists") - - # Shuffle to avoid always scanning same order - random.shuffle(artists_to_scan) - - logger.info(f"Total artists to scan this run: {len(artists_to_scan)}") - if len(all_watchlist_artists) > max_artists_per_scan: - logger.info(f"Skipping {len(all_watchlist_artists) - len(artists_to_scan)} artists (will be scanned in future runs)") - - watchlist_artists = artists_to_scan - - # PROACTIVE ID BACKFILLING (cross-provider support) - # Before scanning, ensure ALL artists have IDs for ALL available sources - # iTunes and Deezer are always available; Spotify requires authentication - if self.spotify_client and self.spotify_client.is_rate_limited(): - self._disable_spotify_for_run("global Spotify rate limit active") - providers_to_backfill = ['itunes', 'deezer'] - if self._spotify_is_primary_source(): - providers_to_backfill.append('spotify') - try: - from config.settings import config_manager as _cfg - if _cfg.get('discogs.token', ''): - providers_to_backfill.append('discogs') - except Exception: - pass - - for provider in providers_to_backfill: - try: - self._backfill_missing_ids(all_watchlist_artists, provider) - except Exception as backfill_error: - logger.warning(f"Error during {provider} ID backfilling: {backfill_error}") - # Continue with scan even if backfilling fails - - scan_results = [] - for i, artist in enumerate(watchlist_artists): - if self.spotify_client and self.spotify_client.is_rate_limited(): - self._disable_spotify_for_run("global Spotify rate limit active") - - try: - result = self.scan_artist(artist) - scan_results.append(result) - if self.spotify_client and self.spotify_client.is_rate_limited(): - self._disable_spotify_for_run("global Spotify rate limit active") - - if result.success: - logger.info(f"Scanned {artist.artist_name}: {result.new_tracks_found} new tracks found") - else: - logger.warning(f"Failed to scan {artist.artist_name}: {result.error_message}") - - # Rate limiting: Add delay between artists to avoid hitting Spotify API limits - # This is critical to prevent getting banned for 6+ hours - if i < len(watchlist_artists) - 1: # Don't delay after the last artist - logger.debug(f"Rate limiting: waiting {DELAY_BETWEEN_ARTISTS}s before scanning next artist") - time.sleep(DELAY_BETWEEN_ARTISTS) - - except Exception as e: - logger.error(f"Error scanning artist {artist.artist_name}: {e}") - scan_results.append(ScanResult( - artist_name=artist.artist_name, - spotify_artist_id=artist.spotify_artist_id, - albums_checked=0, - new_tracks_found=0, - tracks_added_to_wishlist=0, - success=False, - error_message=str(e) - )) - - # Log summary - successful_scans = [r for r in scan_results if r.success] - total_new_tracks = sum(r.new_tracks_found for r in successful_scans) - total_added_to_wishlist = sum(r.tracks_added_to_wishlist for r in successful_scans) - - logger.info(f"Watchlist scan complete: {len(successful_scans)}/{len(scan_results)} artists scanned successfully") - logger.info(f"Found {total_new_tracks} new tracks, added {total_added_to_wishlist} to wishlist") - - # Populate discovery pool with tracks from similar artists - logger.info("Starting discovery pool population...") - if self.spotify_client and self.spotify_client.is_rate_limited(): - self._disable_spotify_for_run("global Spotify rate limit active") - self.populate_discovery_pool() - - # Populate seasonal content (runs independently with its own threshold) - logger.info("Updating seasonal content...") - self._populate_seasonal_content() - - # Generate Last.fm Radio playlists for top tracks (max once per week) - self._generate_lastfm_radio_playlists() - - # Sync Spotify library cache (runs after main scan) - try: - if self.spotify_client and self.spotify_client.is_rate_limited(): - self._disable_spotify_for_run("global Spotify rate limit active") - self.sync_spotify_library_cache() - except Exception as lib_err: - logger.warning(f"Error syncing Spotify library cache: {lib_err}") - - return scan_results - - except Exception as e: - logger.error(f"Error during watchlist scan: {e}") - return [] - finally: - self._reset_spotify_run_state() - - def scan_artist(self, watchlist_artist: WatchlistArtist) -> ScanResult: - """ - Scan a single artist for new releases. - Only checks releases after the last scan timestamp. - Uses the active provider (Spotify if authenticated, otherwise iTunes). - """ - try: - logger.info(f"Scanning artist: {watchlist_artist.artist_name}") - - # Get the active client and artist ID based on provider - client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist) - - if client is None or artist_id is None: - return ScanResult( - artist_name=watchlist_artist.artist_name, - spotify_artist_id=watchlist_artist.spotify_artist_id or '', - albums_checked=0, - new_tracks_found=0, - tracks_added_to_wishlist=0, - success=False, - error_message=f"No {self.metadata_service.get_active_provider()} ID available for this artist" - ) - - logger.info(f"Using {provider} provider for {watchlist_artist.artist_name} (ID: {artist_id})") - - # Update artist image if missing or on every scan to keep fresh - try: - image_url = None - artist_data = client.get_artist(artist_id) - if artist_data: - if 'images' in artist_data and artist_data['images']: - # Spotify/Deezer format: array of {url, height, width} - image_url = artist_data['images'][1]['url'] if len(artist_data['images']) > 1 else artist_data['images'][0]['url'] - elif artist_data.get('image_url'): - # Direct image_url format (iTunes/some providers) - image_url = artist_data['image_url'] - - if image_url: - db_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or watchlist_artist.deezer_artist_id or artist_id - self.database.update_watchlist_artist_image(db_artist_id, image_url) - if not watchlist_artist.image_url: - logger.info(f"Backfilled artist image for {watchlist_artist.artist_name}") - else: - logger.debug(f"No image available for {watchlist_artist.artist_name} from {provider}") - except Exception as img_error: - logger.warning(f"Could not update artist image for {watchlist_artist.artist_name}: {img_error}") - - # Get artist discography using active provider - albums = self._get_artist_discography_with_client(client, artist_id, watchlist_artist.last_scan_timestamp, lookback_days=watchlist_artist.lookback_days) - - if albums is None: - return ScanResult( - artist_name=watchlist_artist.artist_name, - spotify_artist_id=watchlist_artist.spotify_artist_id or '', - albums_checked=0, - new_tracks_found=0, - tracks_added_to_wishlist=0, - success=False, - error_message=f"Failed to get artist discography from {provider}" - ) - - logger.info(f"Found {len(albums)} albums/singles to check for {watchlist_artist.artist_name}") - - # Safety check: Limit number of albums to scan to prevent extremely long sessions - MAX_ALBUMS_PER_ARTIST = 50 # Reasonable limit to prevent API abuse - if len(albums) > MAX_ALBUMS_PER_ARTIST: - logger.warning(f"Artist {watchlist_artist.artist_name} has {len(albums)} albums, limiting to {MAX_ALBUMS_PER_ARTIST} most recent") - albums = albums[:MAX_ALBUMS_PER_ARTIST] # Most recent albums are first - - # Check each album/single for missing tracks - new_tracks_found = 0 - tracks_added_to_wishlist = 0 - - for album_index, album in enumerate(albums): - try: - # Get full album data - logger.info(f"Checking album {album_index + 1}/{len(albums)}: {album.name}") - album_data = client.get_album(album.id) - if not album_data: - continue - - # Get album tracks (works for both Spotify and iTunes) - # Spotify's get_album() includes tracks, but we use get_album_tracks() for consistency - tracks_data = client.get_album_tracks(album.id) - if not tracks_data or not tracks_data.get('items'): - continue - - tracks = tracks_data['items'] - logger.debug(f"Checking album: {album_data.get('name', 'Unknown')} ({len(tracks)} tracks)") - - # Check if user wants this type of release - if not self._should_include_release(len(tracks), watchlist_artist): - release_type = "album" if len(tracks) >= 7 else ("EP" if len(tracks) >= 4 else "single") - logger.debug(f"Skipping {release_type}: {album_data.get('name', 'Unknown')} - user preference") - continue - - # Skip albums with placeholder track names (unreleased tracklist) - # Spotify uses "Track 1", "Track 2", etc. for unannounced tracks - if self._has_placeholder_tracks(tracks): - logger.info(f"Skipping album with placeholder tracks (unreleased tracklist): {album_data.get('name', 'Unknown')}") - continue - - # Check each track - for track in tracks: - # Check content type filters (live, remix, acoustic, compilation) - if not self._should_include_track(track, album_data, watchlist_artist): - continue # Skip this track based on content type preferences - - if self.is_track_missing_from_library(track, album_name=album_data.get('name')): - new_tracks_found += 1 - - # Add to wishlist - if self.add_track_to_wishlist(track, album_data, watchlist_artist): - tracks_added_to_wishlist += 1 - - # Rate limiting: Add delay between albums to prevent API abuse - # This is especially important for artists with many albums - if album_index < len(albums) - 1: # Don't delay after the last album - logger.debug(f"Rate limiting: waiting {DELAY_BETWEEN_ALBUMS}s before next album") - time.sleep(DELAY_BETWEEN_ALBUMS) - - except Exception as e: - logger.warning(f"Error checking album {album.name}: {e}") - continue - - # Update last scan timestamp for this artist - self.update_artist_scan_timestamp(watchlist_artist) - - # Fetch and store similar artists for discovery feature (with caching to avoid over-polling) - # Similar artists are fetched from MusicMap (works with any source) and matched to both Spotify and iTunes - source_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or str(watchlist_artist.id) - try: - # Check if we have fresh similar artists cached (< 30 days old) - # If Spotify is authenticated, also require Spotify IDs to be present - spotify_authenticated = self.spotify_client and self.spotify_client.is_spotify_authenticated() - artist_profile_id = getattr(watchlist_artist, 'profile_id', 1) - if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=artist_profile_id): - logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping MusicMap fetch") - # Even if cached, backfill missing iTunes IDs (seamless dual-source support) - self._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=artist_profile_id) - else: - logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}...") - self.update_similar_artists(watchlist_artist, profile_id=artist_profile_id) - logger.info(f"Similar artists updated for {watchlist_artist.artist_name}") - except Exception as similar_error: - logger.warning(f"Failed to update similar artists for {watchlist_artist.artist_name}: {similar_error}") - - return ScanResult( - artist_name=watchlist_artist.artist_name, - spotify_artist_id=watchlist_artist.spotify_artist_id or '', - albums_checked=len(albums), - new_tracks_found=new_tracks_found, - tracks_added_to_wishlist=tracks_added_to_wishlist, - success=True - ) - - except Exception as e: - logger.error(f"Error scanning artist {watchlist_artist.artist_name}: {e}") - return ScanResult( - artist_name=watchlist_artist.artist_name, - spotify_artist_id=watchlist_artist.spotify_artist_id or '', - albums_checked=0, - new_tracks_found=0, - tracks_added_to_wishlist=0, - success=False, - error_message=str(e) - ) - def _apply_global_watchlist_overrides(self, watchlist_artists: List[WatchlistArtist]): """Apply global watchlist release-type overrides to a batch of artists.""" try: diff --git a/web_server.py b/web_server.py index f74db392..58c08eb1 100644 --- a/web_server.py +++ b/web_server.py @@ -41463,7 +41463,7 @@ def refresh_spotify_library(): def _run_sync(): try: from core.watchlist_scanner import get_watchlist_scanner - scanner = get_watchlist_scanner() + scanner = get_watchlist_scanner(spotify_client) if scanner: # Force full sync by clearing last_sync timestamp database = get_database() From 9d73b8b561ad67ca4766413f0c3785dcbfffd2fb Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Thu, 16 Apr 2026 08:31:04 +0300 Subject: [PATCH 03/10] Restore placeholder filtering and shared image backfill Bring placeholder tracklist skipping back into the shared watchlist scan path, and centralize the DB-only artist image backfill helper so both web scan entrypoints reuse the same logic. --- core/watchlist_scanner.py | 84 +++++++++++++++++++++++++++ tests/test_watchlist_scanner_scan.py | 58 +++++++++++++++++++ web_server.py | 86 ++++------------------------ 3 files changed, 153 insertions(+), 75 deletions(-) diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 069584b3..88643a83 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -517,6 +517,87 @@ class WatchlistScanner: return None + def backfill_watchlist_artist_images(self, profile_id: int) -> int: + """Backfill missing watchlist artist images using cached metadata and existing album art.""" + try: + conn = self.database._get_connection() + cursor = conn.cursor() + cursor.execute(""" + SELECT id, artist_name, spotify_artist_id, itunes_artist_id, + deezer_artist_id, discogs_artist_id + FROM watchlist_artists + WHERE profile_id = ? AND (image_url IS NULL OR image_url = '' OR image_url = 'None' + OR image_url NOT LIKE 'http%') + """, (profile_id,)) + imageless = cursor.fetchall() + + if not imageless: + return 0 + + logger.info("Backfilling images for %s watchlist artists (profile %s)...", len(imageless), profile_id) + filled = 0 + for row in imageless: + name = row['artist_name'] + img = None + + # 1. Check metadata cache for artist image + cursor.execute(""" + SELECT image_url FROM metadata_cache_entities + WHERE entity_type = 'artist' AND name = ? COLLATE NOCASE + AND image_url IS NOT NULL AND image_url LIKE 'http%' + LIMIT 1 + """, (name,)) + cr = cursor.fetchone() + if cr: + img = cr['image_url'] + + # 2. Deezer direct URL (no API call needed) + if not img and row['deezer_artist_id']: + img = f"https://api.deezer.com/artist/{row['deezer_artist_id']}/image?size=big" + + # 3. Deezer ID from cache (artist may have a Deezer match we haven't stored on watchlist) + if not img: + cursor.execute(""" + SELECT entity_id FROM metadata_cache_entities + WHERE entity_type = 'artist' AND source = 'deezer' + AND name = ? COLLATE NOCASE LIMIT 1 + """, (name,)) + dz = cursor.fetchone() + if dz and dz['entity_id']: + img = f"https://api.deezer.com/artist/{dz['entity_id']}/image?size=big" + + # 4. Album art fallback (iTunes artists have no artist images) + if not img: + cursor.execute(""" + SELECT image_url FROM metadata_cache_entities + WHERE entity_type = 'album' AND image_url LIKE 'http%' + AND artist_name = ? COLLATE NOCASE LIMIT 1 + """, (name,)) + alb = cursor.fetchone() + if alb: + img = alb['image_url'] + + if img: + aid = (row['spotify_artist_id'] or row['itunes_artist_id'] + or row['deezer_artist_id'] or row['discogs_artist_id']) + if aid: + self.database.update_watchlist_artist_image(aid, img) + else: + # No external IDs — update by internal row ID directly + cursor.execute(""" + UPDATE watchlist_artists SET image_url = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, (img, row['id'])) + conn.commit() + filled += 1 + + if filled: + logger.info("Backfilled %s/%s watchlist artist images (profile %s)", filled, len(imageless), profile_id) + return filled + except Exception as e: + logger.debug("Error backfilling watchlist artist images for profile %s: %s", profile_id, e, exc_info=True) + return 0 + def get_artist_discography_for_watchlist(self, watchlist_artist: WatchlistArtist, last_scan_timestamp: Optional[datetime] = None) -> Optional[List]: """ Get artist's discography using the active provider, with proper ID resolution. @@ -836,6 +917,9 @@ class WatchlistScanner: continue tracks = album_data['tracks']['items'] + if self._has_placeholder_tracks(tracks): + logger.info("Skipping album with placeholder tracks: %s", album_data.get('name', album.name)) + continue if not self._should_include_release(len(tracks), artist): continue diff --git a/tests/test_watchlist_scanner_scan.py b/tests/test_watchlist_scanner_scan.py index bb59d786..ca99d179 100644 --- a/tests/test_watchlist_scanner_scan.py +++ b/tests/test_watchlist_scanner_scan.py @@ -157,6 +157,64 @@ def test_scan_watchlist_artists_scans_tracks_and_updates_state(monkeypatch): assert scan_state["recent_wishlist_additions"][0]["track_name"] == "Track One" +def test_scan_watchlist_artists_skips_placeholder_tracklists(monkeypatch): + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0) + monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0) + + artist = _build_artist() + album = types.SimpleNamespace(id="album-1", name="Album One") + album_data = { + "name": "Album One", + "images": [{"url": "https://example.com/album.jpg"}], + "tracks": { + "items": [ + { + "id": "track-1", + "name": "Track 1", + "track_number": 1, + "disc_number": 1, + "artists": [{"name": "Artist One"}], + }, + { + "id": "track-2", + "name": "Track 2", + "track_number": 2, + "disc_number": 1, + "artists": [{"name": "Artist One"}], + }, + ] + }, + } + scanner = _build_scanner(album_data, [artist]) + scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False + + monkeypatch.setattr(scanner, "_backfill_missing_ids", lambda *args, **kwargs: None) + monkeypatch.setattr(scanner, "get_artist_image_url", lambda *_args, **_kwargs: "https://example.com/artist.jpg") + monkeypatch.setattr(scanner, "get_artist_discography_for_watchlist", lambda *_args, **_kwargs: [album]) + monkeypatch.setattr(scanner, "_get_lookback_period_setting", lambda: "30") + monkeypatch.setattr(scanner, "_get_rescan_cutoff", lambda: None) + monkeypatch.setattr(scanner, "_should_include_release", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_should_include_track", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "is_track_missing_from_library", lambda *_args, **_kwargs: True) + + add_calls = [] + monkeypatch.setattr(scanner, "add_track_to_wishlist", lambda *args, **kwargs: add_calls.append((args, kwargs)) or True) + monkeypatch.setattr(scanner, "update_artist_scan_timestamp", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "update_similar_artists", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scanner, "_backfill_similar_artists_itunes_ids", lambda *_args, **_kwargs: 0) + + scan_state = {} + results = scanner.scan_watchlist_artists([artist], scan_state=scan_state) + + assert len(results) == 1 + assert results[0].success is True + assert results[0].new_tracks_found == 0 + assert results[0].tracks_added_to_wishlist == 0 + assert add_calls == [] + assert scan_state["summary"]["new_tracks_found"] == 0 + assert scan_state["summary"]["tracks_added_to_wishlist"] == 0 + + def test_scan_watchlist_artists_honors_cancel_check(monkeypatch): monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0) monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0) diff --git a/web_server.py b/web_server.py index 58c08eb1..f6323a2c 100644 --- a/web_server.py +++ b/web_server.py @@ -39836,82 +39836,10 @@ def start_watchlist_scan(): except Exception as backfill_error: print(f"Error during {_bf_provider} ID backfilling: {backfill_error}") # Continue with next provider - - # IMAGE BACKFILL — fix watchlist artists with missing images - # Uses DB-only lookups (metadata cache + album art) — no API calls try: - conn = database._get_connection() - cursor = conn.cursor() - cursor.execute(""" - SELECT id, artist_name, spotify_artist_id, itunes_artist_id, - deezer_artist_id, discogs_artist_id - FROM watchlist_artists - WHERE profile_id = ? AND (image_url IS NULL OR image_url = '' OR image_url = 'None' - OR image_url NOT LIKE 'http%') - """, (scan_profile_id,)) - imageless = cursor.fetchall() - - if imageless: - print(f"Backfilling images for {len(imageless)} watchlist artists...") - filled = 0 - for row in imageless: - name = row['artist_name'] - nn = name.lower().strip() - img = None - - # 1. Check metadata cache for artist image - cursor.execute(""" - SELECT image_url FROM metadata_cache_entities - WHERE entity_type = 'artist' AND name = ? COLLATE NOCASE - AND image_url IS NOT NULL AND image_url LIKE 'http%' - LIMIT 1 - """, (name,)) - cr = cursor.fetchone() - if cr: - img = cr['image_url'] - - # 2. Deezer direct URL (no API call needed) - if not img and row['deezer_artist_id']: - img = f"https://api.deezer.com/artist/{row['deezer_artist_id']}/image?size=big" - - # 3. Deezer ID from cache (artist may have a Deezer match we haven't stored on watchlist) - if not img: - cursor.execute(""" - SELECT entity_id FROM metadata_cache_entities - WHERE entity_type = 'artist' AND source = 'deezer' - AND name = ? COLLATE NOCASE LIMIT 1 - """, (name,)) - dz = cursor.fetchone() - if dz and dz['entity_id']: - img = f"https://api.deezer.com/artist/{dz['entity_id']}/image?size=big" - - # 4. Album art fallback (iTunes artists have no artist images) - if not img: - cursor.execute(""" - SELECT image_url FROM metadata_cache_entities - WHERE entity_type = 'album' AND image_url LIKE 'http%' - AND artist_name = ? COLLATE NOCASE LIMIT 1 - """, (name,)) - alb = cursor.fetchone() - if alb: - img = alb['image_url'] - - if img: - aid = (row['spotify_artist_id'] or row['itunes_artist_id'] - or row['deezer_artist_id'] or row['discogs_artist_id']) - if aid: - database.update_watchlist_artist_image(aid, img) - else: - # No external IDs — update by internal row ID directly - cursor.execute(""" - UPDATE watchlist_artists SET image_url = ?, updated_at = CURRENT_TIMESTAMP - WHERE id = ? - """, (img, row['id'])) - conn.commit() - filled += 1 - - if filled: - print(f"Backfilled {filled}/{len(imageless)} watchlist artist images") + filled = scanner.backfill_watchlist_artist_images(scan_profile_id) + if filled: + print(f"Backfilled {filled} watchlist artist images") except Exception as img_err: print(f"Image backfill error: {img_err}") @@ -40703,6 +40631,14 @@ def _process_watchlist_scan_automatically(automation_id=None, profile_id=None): scanner = get_watchlist_scanner(spotify_client) all_profiles = scan_profiles # Used later for discovery pool population + for p in scan_profiles: + try: + filled = scanner.backfill_watchlist_artist_images(p['id']) + if filled: + print(f"Backfilled {filled} watchlist artist images for profile {p['id']}") + except Exception as img_err: + print(f"Image backfill error for profile {p['id']}: {img_err}") + # Initialize detailed progress tracking (same as manual scan) watchlist_scan_state = { 'status': 'scanning', From 40f39604ad3c06f466fd489806807eb96e192666 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Thu, 16 Apr 2026 08:35:27 +0300 Subject: [PATCH 04/10] Restore Last.fm radio after watchlist scans Keep the weekly Last.fm radio generation step in the web watchlist scan post-processing chain so the higher-level scan behavior stays intact after moving the scan loop into the shared scanner core. --- web_server.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/web_server.py b/web_server.py index f6323a2c..87d157ca 100644 --- a/web_server.py +++ b/web_server.py @@ -39961,6 +39961,15 @@ def start_watchlist_scan(): import traceback traceback.print_exc() + # Generate Last.fm radio playlists (weekly refresh) + print("Starting Last.fm radio generation...") + watchlist_scan_state['current_phase'] = 'generating_lastfm_radio' + try: + scanner._generate_lastfm_radio_playlists() + print("Last.fm radio generation complete") + except Exception as lastfm_error: + print(f"Error generating Last.fm radio playlists: {lastfm_error}") + # Sync Spotify library cache print("Syncing Spotify library cache...") watchlist_scan_state['current_phase'] = 'syncing_spotify_library' @@ -40875,6 +40884,21 @@ def _process_watchlist_scan_automatically(automation_id=None, profile_id=None): _update_automation_progress(automation_id, log_line=f'Seasonal error: {seasonal_error}', log_type='error') + # Generate Last.fm radio playlists (weekly refresh) + print("Starting Last.fm radio generation...") + watchlist_scan_state['current_phase'] = 'generating_lastfm_radio' + _update_automation_progress(automation_id, progress=99, phase='Generating Last.fm radio', + log_line='Building Last.fm radio playlists...', log_type='info') + try: + scanner._generate_lastfm_radio_playlists() + print("Last.fm radio generation complete") + _update_automation_progress(automation_id, + log_line='Last.fm radio playlists updated', log_type='success') + except Exception as lastfm_error: + print(f"Error generating Last.fm radio playlists: {lastfm_error}") + _update_automation_progress(automation_id, + log_line=f'Last.fm radio error: {lastfm_error}', log_type='error') + # Sync Spotify library cache print("Syncing Spotify library cache...") try: From 38b907097d88d46b239d24188c9a5458e91323b3 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Thu, 16 Apr 2026 09:13:15 +0300 Subject: [PATCH 05/10] Make watchlist scanning source-aware Move the web watchlist scan core onto the shared metadata source priority so primary provider settings are respected during artist, album, and image resolution. Add coverage for primary-source-first discography lookup and fallback to later providers when the primary source has no albums. --- core/watchlist_scanner.py | 428 ++++++++++++++++++++------- tests/test_watchlist_scanner_scan.py | 119 ++++++++ 2 files changed, 443 insertions(+), 104 deletions(-) diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 88643a83..6f3d1983 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -13,6 +13,12 @@ import requests from bs4 import BeautifulSoup from database.music_database import get_database, WatchlistArtist from core.spotify_client import SpotifyClient +from core.metadata_service import ( + get_album_tracks_for_source, + get_client_for_source, + get_primary_source, + get_source_priority, +) from core.wishlist_service import get_wishlist_service from core.matching_engine import MusicMatchingEngine from utils.logging_config import get_logger @@ -344,6 +350,16 @@ class ScanResult: success: bool error_message: Optional[str] = None + +@dataclass +class WatchlistDiscographyResult: + """Resolved watchlist artist discography for a specific metadata source.""" + source: str + client: Any + artist_id: str + albums: List[Any] + image_url: Optional[str] = None + class WatchlistScanner: """Service for scanning watched artists for new releases""" @@ -428,11 +444,210 @@ class WatchlistScanner: if not self._spotify_available_for_run(): return False try: - from core.metadata_service import get_primary_source return get_primary_source() == 'spotify' except Exception: return False + def _watchlist_source_priority(self) -> List[str]: + """Return watchlist scan sources in the configured priority order.""" + return list(get_source_priority(get_primary_source())) + + @staticmethod + def _artist_id_attribute_for_source(source: str) -> Optional[str]: + """Return the watchlist artist attribute that stores the given source ID.""" + return { + 'spotify': 'spotify_artist_id', + 'itunes': 'itunes_artist_id', + 'deezer': 'deezer_artist_id', + 'discogs': 'discogs_artist_id', + }.get(source) + + @staticmethod + def _extract_entity_id(value: Any) -> Optional[str]: + """Extract an ID from a dataclass, dict, or plain object.""" + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, dict): + return value.get('id') or value.get('artist_id') or value.get('release_id') + return getattr(value, 'id', None) or getattr(value, 'artist_id', None) or getattr(value, 'release_id', None) + + def _cache_watchlist_artist_source_id(self, watchlist_artist: WatchlistArtist, source: str, source_id: str) -> None: + """Cache a resolved artist ID for a watchlist artist when we have a storage column.""" + if not source_id: + return + + if source == 'spotify': + self.database.update_watchlist_spotify_id(watchlist_artist.id, source_id) + watchlist_artist.spotify_artist_id = source_id + elif source == 'itunes': + self.database.update_watchlist_itunes_id(watchlist_artist.id, source_id) + watchlist_artist.itunes_artist_id = source_id + elif source == 'deezer': + self.database.update_watchlist_deezer_id(watchlist_artist.id, source_id) + watchlist_artist.deezer_artist_id = source_id + elif source == 'discogs': + self.database.update_watchlist_discogs_id(watchlist_artist.id, source_id) + watchlist_artist.discogs_artist_id = source_id + + def _resolve_watchlist_artist_source_id(self, watchlist_artist: WatchlistArtist, source: str, client: Any) -> Optional[str]: + """Resolve the artist ID for an exact source, searching by name if needed.""" + attr = self._artist_id_attribute_for_source(source) + stored_id = getattr(watchlist_artist, attr, None) if attr else None + if stored_id: + return stored_id + + if not client or not hasattr(client, 'search_artists'): + return None + + try: + search_results = client.search_artists(watchlist_artist.artist_name, limit=1) + except Exception as e: + logger.debug("Could not search %s for %s: %s", source, watchlist_artist.artist_name, e) + return None + + if not search_results: + return None + + found_id = self._extract_entity_id(search_results[0]) + if found_id and attr: + self._cache_watchlist_artist_source_id(watchlist_artist, source, found_id) + return found_id + + @staticmethod + def _get_artist_image_from_data(artist_data: Any) -> Optional[str]: + """Extract an image URL from artist payloads across providers.""" + if not artist_data: + return None + + if isinstance(artist_data, dict): + images = artist_data.get('images') or [] + if images: + first_image = images[0] + if isinstance(first_image, dict): + return first_image.get('url') + return ( + artist_data.get('image_url') + or artist_data.get('thumb_url') + or artist_data.get('cover_image') + or artist_data.get('picture_xl') + or artist_data.get('picture_big') + or artist_data.get('picture_medium') + ) + + images = getattr(artist_data, 'images', None) + if images: + first_image = images[0] + if isinstance(first_image, dict): + return first_image.get('url') + return ( + getattr(artist_data, 'image_url', None) + or getattr(artist_data, 'thumb_url', None) + or getattr(artist_data, 'cover_image', None) + ) + + def _get_artist_image_for_source(self, watchlist_artist: WatchlistArtist, source: str, client: Any, artist_id: str) -> Optional[str]: + """Fetch an artist image for a specific source.""" + if not client or not artist_id or not hasattr(client, 'get_artist'): + return None + + try: + artist_data = client.get_artist(artist_id) + except Exception as e: + logger.debug("Could not fetch artist image for %s on %s: %s", watchlist_artist.artist_name, source, e) + return None + + return self._get_artist_image_from_data(artist_data) + + def _get_album_data_for_source(self, source: str, client: Any, album_id: str, album_name: str = '') -> Optional[Dict[str, Any]]: + """Fetch album data for a specific source and normalize track payloads when needed.""" + if not client or not album_id or not hasattr(client, 'get_album'): + return None + + try: + album_data = client.get_album(album_id) + except Exception as e: + logger.debug("Could not fetch album %s on %s: %s", album_id, source, e) + album_data = None + + if not album_data: + return None + + # Some providers return album metadata without embedded tracks; normalize that shape. + tracks = album_data.get('tracks') if isinstance(album_data, dict) else None + if not tracks: + track_items = get_album_tracks_for_source(source, album_id) + if track_items: + if not isinstance(album_data, dict): + try: + album_data = dict(album_data) + except Exception: + album_data = {'name': album_name or album_id} + if isinstance(track_items, dict): + album_data['tracks'] = track_items + else: + album_data['tracks'] = {'items': track_items} + + return album_data + + @staticmethod + def _extract_track_items(album_data: Any) -> List[Dict[str, Any]]: + """Normalize track payloads from different album formats to a list of items.""" + if not album_data: + return [] + + tracks = None + if isinstance(album_data, dict): + tracks = album_data.get('tracks') + else: + tracks = getattr(album_data, 'tracks', None) + + if not tracks: + return [] + + if isinstance(tracks, dict): + items = tracks.get('items') or tracks.get('data') or [] + return list(items) if isinstance(items, list) else [] + + if isinstance(tracks, list): + return tracks + + return [] + + def _resolve_watchlist_discography_for_source( + self, + watchlist_artist: WatchlistArtist, + source: str, + last_scan_timestamp: Optional[datetime] = None, + ) -> Optional[WatchlistDiscographyResult]: + """Resolve a watchlist artist to a specific source and fetch its discography.""" + client = get_client_for_source(source) + if not client: + return None + + artist_id = self._resolve_watchlist_artist_source_id(watchlist_artist, source, client) + if not artist_id: + return None + + albums = self._get_artist_discography_with_client( + client, + artist_id, + last_scan_timestamp, + lookback_days=watchlist_artist.lookback_days, + ) + if not albums: + return None + + image_url = self._get_artist_image_for_source(watchlist_artist, source, client, artist_id) + return WatchlistDiscographyResult( + source=source, + client=client, + artist_id=artist_id, + albums=albums, + image_url=image_url, + ) + def _get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist): """ Get the appropriate client and artist ID based on active provider. @@ -495,26 +710,21 @@ class WatchlistScanner: def get_artist_image_url(self, watchlist_artist: WatchlistArtist) -> Optional[str]: """ - Get artist image URL using the active provider. + Get artist image URL using the configured source priority. Returns: Image URL string or None if not available """ - client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist) - if not client or not artist_id: - return None - - try: - artist_data = client.get_artist(artist_id) - if artist_data: - # Handle both Spotify and iTunes response formats - if 'images' in artist_data and artist_data['images']: - return artist_data['images'][0].get('url') - elif 'image_url' in artist_data: - return artist_data['image_url'] - except Exception as e: - logger.debug(f"Could not fetch artist image for {watchlist_artist.artist_name}: {e}") - + for source in self._watchlist_source_priority(): + client = get_client_for_source(source) + if not client: + continue + artist_id = self._resolve_watchlist_artist_source_id(watchlist_artist, source, client) + if not artist_id: + continue + image_url = self._get_artist_image_for_source(watchlist_artist, source, client, artist_id) + if image_url: + return image_url return None def backfill_watchlist_artist_images(self, profile_id: int) -> int: @@ -598,57 +808,25 @@ class WatchlistScanner: logger.debug("Error backfilling watchlist artist images for profile %s: %s", profile_id, e, exc_info=True) return 0 - def get_artist_discography_for_watchlist(self, watchlist_artist: WatchlistArtist, last_scan_timestamp: Optional[datetime] = None) -> Optional[List]: + def get_artist_discography_for_watchlist(self, watchlist_artist: WatchlistArtist, last_scan_timestamp: Optional[datetime] = None) -> Optional[WatchlistDiscographyResult]: """ - Get artist's discography using the active provider, with proper ID resolution. - This is the provider-aware version of get_artist_discography. + Get artist's discography using the configured source priority, with proper ID resolution. + Returns the first provider that can actually return albums. Args: watchlist_artist: WatchlistArtist object (has both spotify and itunes IDs) last_scan_timestamp: Only return releases after this date (for incremental scans) Returns: - List of albums or None on error + WatchlistDiscographyResult or None on error """ - client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist) - if not client or not artist_id: - logger.warning(f"No valid client/ID for {watchlist_artist.artist_name}") - return None + for source in self._watchlist_source_priority(): + result = self._resolve_watchlist_discography_for_source(watchlist_artist, source, last_scan_timestamp) + if result: + return result - albums = self._get_artist_discography_with_client(client, artist_id, last_scan_timestamp, lookback_days=watchlist_artist.lookback_days) - - # If primary provider returned nothing, try the other provider as fallback - if not albums: - fallback_id = None - fallback_client = None - - if provider == 'spotify': - fallback_client = self.metadata_service.itunes - fallback_id = watchlist_artist.itunes_artist_id - # If no iTunes ID stored, search by name and cache it - if not fallback_id: - try: - search_results = fallback_client.search_artists(watchlist_artist.artist_name, limit=1) - if search_results: - fallback_id = search_results[0].id - logger.info(f"Resolved iTunes ID {fallback_id} for {watchlist_artist.artist_name}") - self.database.update_watchlist_artist_itunes_id( - watchlist_artist.spotify_artist_id or str(watchlist_artist.id), - fallback_id - ) - watchlist_artist.itunes_artist_id = fallback_id - except Exception as e: - logger.debug(f"Could not resolve iTunes ID for {watchlist_artist.artist_name}: {e}") - - elif provider == 'itunes': - fallback_client = self.metadata_service.spotify - fallback_id = watchlist_artist.spotify_artist_id - - if fallback_client and fallback_id: - logger.info(f"{provider.capitalize()} returned no albums for {watchlist_artist.artist_name}, falling back to {'iTunes' if provider == 'spotify' else 'Spotify'}") - albums = self._get_artist_discography_with_client(fallback_client, fallback_id, last_scan_timestamp, lookback_days=watchlist_artist.lookback_days) - - return albums + logger.warning(f"No valid client/ID for {watchlist_artist.artist_name}") + return None def _apply_global_watchlist_overrides(self, watchlist_artists: List[WatchlistArtist]): """Apply global watchlist release-type overrides to a batch of artists.""" @@ -796,15 +974,13 @@ class WatchlistScanner: _emit('scan_started', profile_id=profile_id, total_artists=len(watchlist_artists)) # Rate-limit-aware backfill of artist IDs for the providers we can safely resolve. - providers_to_backfill = ['itunes', 'deezer'] - if self.spotify_client and self.spotify_client.is_spotify_authenticated(): - providers_to_backfill.append('spotify') - try: - from config.settings import config_manager as _cfg - if _cfg.get('discogs.token', ''): - providers_to_backfill.append('discogs') - except Exception: - pass + providers_to_backfill = [] + for source in self._watchlist_source_priority(): + if source == 'spotify': + if self._spotify_available_for_run(): + providers_to_backfill.append(source) + elif source in {'itunes', 'deezer', 'discogs'} and get_client_for_source(source): + providers_to_backfill.append(source) for provider in providers_to_backfill: try: @@ -855,8 +1031,46 @@ class WatchlistScanner: _emit('cancelled', processed=i, total=len(watchlist_artists)) break + source_artist_id = ( + artist.spotify_artist_id + or artist.itunes_artist_id + or artist.deezer_artist_id + or artist.discogs_artist_id + or str(artist.id) + ) + try: - artist_image_url = self.get_artist_image_url(artist) or '' + discography_result = self.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp) + if discography_result is None: + scan_results.append(ScanResult( + artist_name=artist.artist_name, + spotify_artist_id=source_artist_id, + albums_checked=0, + new_tracks_found=0, + tracks_added_to_wishlist=0, + success=False, + error_message="Failed to get artist discography", + )) + _emit( + 'artist_error', + artist_name=artist.artist_name, + profile_id=profile_id, + error_message="Failed to get artist discography", + ) + continue + + if isinstance(discography_result, list): + albums = discography_result + artist_image_url = self.get_artist_image_url(artist) or '' + album_fetcher = lambda album_id, album_name='': self.metadata_service.get_album(album_id) + else: + source = discography_result.source + client = discography_result.client + albums = discography_result.albums + source_artist_id = discography_result.artist_id + artist_image_url = discography_result.image_url or self.get_artist_image_url(artist) or '' + album_fetcher = lambda album_id, album_name='': self._get_album_data_for_source(source, client, album_id, album_name) + absolute_index = artist_index_offset + i + 1 if scan_state is not None: scan_state.update({ @@ -880,25 +1094,6 @@ class WatchlistScanner: artist_image_url=artist_image_url, ) - albums = self.get_artist_discography_for_watchlist(artist, artist.last_scan_timestamp) - if albums is None: - scan_results.append(ScanResult( - artist_name=artist.artist_name, - spotify_artist_id=artist.spotify_artist_id or '', - albums_checked=0, - new_tracks_found=0, - tracks_added_to_wishlist=0, - success=False, - error_message="Failed to get artist discography", - )) - _emit( - 'artist_error', - artist_name=artist.artist_name, - profile_id=profile_id, - error_message="Failed to get artist discography", - ) - continue - if scan_state is not None: scan_state.update({ 'current_phase': 'checking_albums', @@ -911,26 +1106,39 @@ class WatchlistScanner: for album_index, album in enumerate(albums): try: - album_data = self.metadata_service.get_album(album.id) - if not album_data or 'tracks' not in album_data: + album_data = album_fetcher(album.id, getattr(album, 'name', '')) + tracks = self._extract_track_items(album_data) + if not album_data or not tracks: logger.debug("Skipping album %s (id=%s): no track data returned", album.name, album.id) continue - tracks = album_data['tracks']['items'] + album_name = getattr(album, 'name', '') + if isinstance(album_data, dict): + album_name = album_data.get('name', album_name) + else: + album_name = getattr(album_data, 'name', album_name) + if self._has_placeholder_tracks(tracks): - logger.info("Skipping album with placeholder tracks: %s", album_data.get('name', album.name)) + logger.info("Skipping album with placeholder tracks: %s", album_name) continue if not self._should_include_release(len(tracks), artist): continue album_image_url = '' - if 'images' in album_data and album_data['images']: - album_image_url = album_data['images'][0]['url'] + album_images = [] + if isinstance(album_data, dict): + album_images = album_data.get('images') or [] + else: + album_images = getattr(album_data, 'images', None) or [] + if album_images: + first_image = album_images[0] + if isinstance(first_image, dict): + album_image_url = first_image.get('url', '') if scan_state is not None: scan_state.update({ 'albums_checked': album_index + 1, - 'current_album': album.name, + 'current_album': album_name, 'current_album_image_url': album_image_url, 'current_phase': f'checking_album_{album_index + 1}_of_{len(albums)}', }) @@ -938,7 +1146,7 @@ class WatchlistScanner: _emit( 'album_started', artist_name=artist.artist_name, - album_name=album.name, + album_name=album_name, album_index=album_index + 1, total_albums=len(albums), album_image_url=album_image_url, @@ -952,7 +1160,7 @@ class WatchlistScanner: if scan_state is not None: scan_state['current_track_name'] = track_name - if self.is_track_missing_from_library(track, album_name=album_data.get('name')): + if self.is_track_missing_from_library(track, album_name=album_name): artist_new_tracks += 1 if scan_state is not None: scan_state['tracks_found_this_scan'] += 1 @@ -984,7 +1192,7 @@ class WatchlistScanner: scan_results.append(ScanResult( artist_name=artist.artist_name, - spotify_artist_id=artist.spotify_artist_id or '', + spotify_artist_id=source_artist_id or artist.spotify_artist_id or '', albums_checked=len(albums), new_tracks_found=artist_new_tracks, tracks_added_to_wishlist=artist_added_tracks, @@ -1005,7 +1213,6 @@ class WatchlistScanner: try: if scan_state is not None: scan_state['current_phase'] = 'fetching_similar_artists' - source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id) artist_profile_id = getattr(artist, 'profile_id', profile_id) spotify_authenticated = self.spotify_client and self.spotify_client.is_spotify_authenticated() if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=artist_profile_id): @@ -1013,7 +1220,7 @@ class WatchlistScanner: self._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=artist_profile_id) else: logger.info("Fetching similar artists for %s (profile %s)...", artist.artist_name, artist_profile_id) - self.update_similar_artists(artist, profile_id=artist_profile_id) + self.update_similar_artists(artist, profile_id=artist_profile_id, source_artist_id=source_artist_id) logger.info("Similar artists updated for %s", artist.artist_name) except Exception as similar_error: logger.warning("Failed to update similar artists for %s: %s", artist.artist_name, similar_error) @@ -1027,7 +1234,7 @@ class WatchlistScanner: logger.error("Error scanning artist %s: %s", artist.artist_name, e) scan_results.append(ScanResult( artist_name=artist.artist_name, - spotify_artist_id=artist.spotify_artist_id or '', + spotify_artist_id=source_artist_id, albums_checked=0, new_tracks_found=0, tracks_added_to_wishlist=0, @@ -2107,7 +2314,13 @@ class WatchlistScanner: logger.error(f"Error backfilling similar artists {fallback_source if 'fallback_source' in dir() else 'fallback'} IDs: {e}") return 0 - def update_similar_artists(self, watchlist_artist: WatchlistArtist, limit: int = 10, profile_id: int = 1) -> bool: + def update_similar_artists( + self, + watchlist_artist: WatchlistArtist, + limit: int = 10, + profile_id: int = 1, + source_artist_id: Optional[str] = None, + ) -> bool: """ Fetch and store similar artists for a watchlist artist. Called after each artist scan to build discovery pool. @@ -2125,8 +2338,15 @@ class WatchlistScanner: logger.info(f"Found {len(similar_artists)} similar artists for {watchlist_artist.artist_name}") - # Use consistent source artist ID (prefer Spotify, fall back to iTunes or internal ID) - source_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or str(watchlist_artist.id) + # Use the ID that matched the scan source when available; otherwise fall back to any known ID. + source_artist_id = ( + source_artist_id + or watchlist_artist.spotify_artist_id + or watchlist_artist.itunes_artist_id + or watchlist_artist.deezer_artist_id + or watchlist_artist.discogs_artist_id + or str(watchlist_artist.id) + ) # Store each similar artist in database stored_count = 0 diff --git a/tests/test_watchlist_scanner_scan.py b/tests/test_watchlist_scanner_scan.py index ca99d179..6b562b22 100644 --- a/tests/test_watchlist_scanner_scan.py +++ b/tests/test_watchlist_scanner_scan.py @@ -22,6 +22,32 @@ if "spotipy" not in sys.modules: sys.modules["spotipy"] = spotipy sys.modules["spotipy.oauth2"] = oauth2 +if "config.settings" not in sys.modules: + config_pkg = types.ModuleType("config") + settings_mod = types.ModuleType("config.settings") + + class _DummyConfigManager: + def get(self, key, default=None): + return default + + def get_active_media_server(self): + return "plex" + + settings_mod.config_manager = _DummyConfigManager() + config_pkg.settings = settings_mod + sys.modules["config"] = config_pkg + sys.modules["config.settings"] = settings_mod + +if "core.matching_engine" not in sys.modules: + matching_engine_mod = types.ModuleType("core.matching_engine") + + class _DummyMatchingEngine: + def clean_title(self, title): + return title + + matching_engine_mod.MusicMatchingEngine = _DummyMatchingEngine + sys.modules["core.matching_engine"] = matching_engine_mod + import core.watchlist_scanner as watchlist_scanner_module from core.watchlist_scanner import WatchlistScanner @@ -41,6 +67,31 @@ class _FakeMetadataService: return self._album_data +class _FakeSourceClient: + def __init__(self, *, artist_id: str, albums, image_url: str): + self.artist_id = artist_id + self.albums = list(albums) + self.image_url = image_url + self.search_calls = [] + self.album_calls = [] + self.artist_calls = [] + + def search_artists(self, query, limit=1): + self.search_calls.append((query, limit)) + return [types.SimpleNamespace(id=self.artist_id, name=query)] + + def get_artist_albums(self, artist_id, album_type='album,single', limit=50, **kwargs): + self.album_calls.append((artist_id, album_type, limit, kwargs)) + return list(self.albums) + + def get_artist(self, artist_id): + self.artist_calls.append(artist_id) + return { + "id": artist_id, + "images": [{"url": self.image_url}] if self.image_url else [], + } + + class _FakeDB: def __init__(self, artists): self.artists = artists @@ -255,3 +306,71 @@ def test_scan_watchlist_artists_honors_cancel_check(monkeypatch): assert scan_state["status"] == "cancelled" assert scan_state["summary"]["cancelled"] is True assert scan_state["summary"]["successful_scans"] == 1 + + +def test_get_artist_discography_for_watchlist_prefers_primary_source(monkeypatch): + monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None)) + monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + + deezer_album = types.SimpleNamespace(id="dz-album", name="Deezer Album", release_date=None) + spotify_album = types.SimpleNamespace(id="sp-album", name="Spotify Album", release_date=None) + + deezer_client = _FakeSourceClient(artist_id="dz-artist", albums=[deezer_album], image_url="https://example.com/deezer.jpg") + spotify_client = _FakeSourceClient(artist_id="sp-artist", albums=[spotify_album], image_url="https://example.com/spotify.jpg") + + def fake_get_client_for_source(source): + return {"deezer": deezer_client, "spotify": spotify_client}.get(source) + + monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source) + + artist = _build_artist() + artist.spotify_artist_id = "sp-artist" + artist.deezer_artist_id = "dz-artist" + + scanner = _build_scanner({"tracks": {"items": []}}, [artist]) + scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False + scanner._get_lookback_period_setting = lambda: "30" + scanner._get_rescan_cutoff = lambda: None + + result = scanner.get_artist_discography_for_watchlist(artist, None) + + assert result is not None + assert result.source == "deezer" + assert result.artist_id == "dz-artist" + assert result.albums and result.albums[0].id == "dz-album" + assert deezer_client.album_calls + assert spotify_client.album_calls == [] + + +def test_get_artist_discography_for_watchlist_falls_back_when_primary_empty(monkeypatch): + monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None)) + monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + + deezer_client = _FakeSourceClient(artist_id="dz-artist", albums=[], image_url="https://example.com/deezer.jpg") + spotify_album = types.SimpleNamespace(id="sp-album", name="Spotify Album", release_date=None) + spotify_client = _FakeSourceClient(artist_id="sp-artist", albums=[spotify_album], image_url="https://example.com/spotify.jpg") + + def fake_get_client_for_source(source): + return {"deezer": deezer_client, "spotify": spotify_client}.get(source) + + monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source) + + artist = _build_artist() + artist.spotify_artist_id = "sp-artist" + artist.deezer_artist_id = "dz-artist" + + scanner = _build_scanner({"tracks": {"items": []}}, [artist]) + scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False + scanner._get_lookback_period_setting = lambda: "30" + scanner._get_rescan_cutoff = lambda: None + + result = scanner.get_artist_discography_for_watchlist(artist, None) + + assert result is not None + assert result.source == "spotify" + assert result.artist_id == "sp-artist" + assert result.albums and result.albums[0].id == "sp-album" + assert deezer_client.album_calls + assert spotify_client.album_calls From 7b3a32ccc5555640a1fca7398e64ba3a9c7d40f9 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Thu, 16 Apr 2026 09:14:07 +0300 Subject: [PATCH 06/10] Remove dead watchlist source helpers Drop the old active-provider artist lookup helpers from watchlist_scanner now that the web scan flow resolves sources through the shared metadata priority. Keep the Spotify-specific feature toggles in place for discovery and sync paths that still use them. --- core/watchlist_scanner.py | 60 --------------------------------------- 1 file changed, 60 deletions(-) diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 6f3d1983..32399b71 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -648,66 +648,6 @@ class WatchlistScanner: image_url=image_url, ) - def _get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist): - """ - Get the appropriate client and artist ID based on active provider. - If iTunes ID is missing, searches by artist name to find and cache it. - - Returns: - Tuple of (client, artist_id, provider_name) or (None, None, None) if no valid ID - """ - provider = self.metadata_service.get_active_provider() - - if provider == 'spotify': - if watchlist_artist.spotify_artist_id: - return (self.metadata_service.spotify, watchlist_artist.spotify_artist_id, 'spotify') - else: - logger.warning(f"No Spotify ID for {watchlist_artist.artist_name}, cannot scan with Spotify") - return (None, None, None) - else: # itunes or deezer fallback - fallback_source = provider # 'itunes' or 'deezer' - fallback_client = self.metadata_service.itunes # May be iTunesClient or DeezerClient - # Pick the right stored ID for the active fallback source - stored_id = watchlist_artist.deezer_artist_id if fallback_source == 'deezer' else watchlist_artist.itunes_artist_id - if stored_id: - return (fallback_client, stored_id, fallback_source) - else: - # No ID stored for this source - search by name and cache it - logger.info(f"No {fallback_source} ID for {watchlist_artist.artist_name}, searching by name...") - try: - search_results = fallback_client.search_artists(watchlist_artist.artist_name, limit=1) - if search_results and len(search_results) > 0: - found_id = search_results[0].id - logger.info(f"Found {fallback_source} ID {found_id} for {watchlist_artist.artist_name}") - # Cache the ID in the database for future use - if fallback_source == 'deezer': - self.database.update_watchlist_artist_deezer_id( - watchlist_artist.spotify_artist_id or str(watchlist_artist.id), - found_id - ) - else: - self.database.update_watchlist_artist_itunes_id( - watchlist_artist.spotify_artist_id or str(watchlist_artist.id), - found_id - ) - return (fallback_client, found_id, fallback_source) - else: - logger.warning(f"Could not find {watchlist_artist.artist_name} on {fallback_source}") - return (None, None, None) - except Exception as e: - logger.error(f"Error searching {fallback_source} for {watchlist_artist.artist_name}: {e}") - return (None, None, None) - - def get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist): - """ - Public wrapper for _get_active_client_and_artist_id. - Gets the appropriate client and artist ID based on active provider. - - Returns: - Tuple of (client, artist_id, provider_name) or (None, None, None) if no valid ID - """ - return self._get_active_client_and_artist_id(watchlist_artist) - def get_artist_image_url(self, watchlist_artist: WatchlistArtist) -> Optional[str]: """ Get artist image URL using the configured source priority. From 09a7465b22a86184ef66dbe40a63a3543df1f603 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Thu, 16 Apr 2026 09:47:36 +0300 Subject: [PATCH 07/10] Add strict fallback opt-out to Spotify client Expose an allow_fallback flag on Spotify search and metadata methods so strict callers can avoid silently resolving through fallback providers. --- core/spotify_client.py | 104 +++++++++++++++++++++++++++-------------- 1 file changed, 70 insertions(+), 34 deletions(-) diff --git a/core/spotify_client.py b/core/spotify_client.py index 8fd09f9d..60d68232 100644 --- a/core/spotify_client.py +++ b/core/spotify_client.py @@ -1098,8 +1098,12 @@ class SpotifyClient: return [] @rate_limited - def search_tracks(self, query: str, limit: int = 10) -> List[Track]: - """Search for tracks - falls back to configured metadata source if Spotify not authenticated""" + def search_tracks(self, query: str, limit: int = 10, allow_fallback: bool = True) -> List[Track]: + """Search for tracks. + + When allow_fallback is True, falls back to the configured metadata source + if Spotify is unavailable or returns an error. + """ cache = get_metadata_cache() use_spotify = self.is_spotify_authenticated() @@ -1146,12 +1150,18 @@ class SpotifyClient: # Fall through to fallback # Fallback (iTunes or Deezer — configured in settings) - logger.debug(f"Using {self._fallback_source} fallback for track search: {query}") - return self._fallback.search_tracks(query, limit) + if allow_fallback: + logger.debug(f"Using {self._fallback_source} fallback for track search: {query}") + return self._fallback.search_tracks(query, limit) + return [] @rate_limited - def search_artists(self, query: str, limit: int = 10) -> List[Artist]: - """Search for artists - falls back to configured metadata source if Spotify not authenticated""" + def search_artists(self, query: str, limit: int = 10, allow_fallback: bool = True) -> List[Artist]: + """Search for artists. + + When allow_fallback is True, falls back to the configured metadata source + if Spotify is unavailable or returns an error. + """ cache = get_metadata_cache() use_spotify = self.is_spotify_authenticated() @@ -1204,15 +1214,21 @@ class SpotifyClient: # Fall through to iTunes fallback # Fallback (iTunes or Deezer) - logger.debug(f"Using {self._fallback_source} fallback for artist search: {query}") - artists = self._fallback.search_artists(query, limit) - query_lower = query.lower().strip() - artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1)) - return artists + if allow_fallback: + logger.debug(f"Using {self._fallback_source} fallback for artist search: {query}") + artists = self._fallback.search_artists(query, limit) + query_lower = query.lower().strip() + artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1)) + return artists + return [] @rate_limited - def search_albums(self, query: str, limit: int = 10) -> List[Album]: - """Search for albums - falls back to configured metadata source if Spotify not authenticated""" + def search_albums(self, query: str, limit: int = 10, allow_fallback: bool = True) -> List[Album]: + """Search for albums. + + When allow_fallback is True, falls back to the configured metadata source + if Spotify is unavailable or returns an error. + """ cache = get_metadata_cache() use_spotify = self.is_spotify_authenticated() @@ -1259,12 +1275,18 @@ class SpotifyClient: # Fall through to iTunes fallback # Fallback (iTunes or Deezer) - logger.debug(f"Using {self._fallback_source} fallback for album search: {query}") - return self._fallback.search_albums(query, limit) + if allow_fallback: + logger.debug(f"Using {self._fallback_source} fallback for album search: {query}") + return self._fallback.search_albums(query, limit) + return [] @rate_limited - def get_track_details(self, track_id: str) -> Optional[Dict[str, Any]]: - """Get detailed track information - falls back to configured metadata source""" + def get_track_details(self, track_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]: + """Get detailed track information. + + When allow_fallback is True, falls back to the configured metadata source + for non-Spotify IDs or Spotify failure. + """ # Check cache — we store raw track_data, reconstruct enhanced on hit cache = get_metadata_cache() fallback_src = self._fallback_source @@ -1277,7 +1299,7 @@ class SpotifyClient: return self._build_enhanced_track(cached) # Simplified track cached by get_album_tracks — treat as cache miss logger.debug(f"Cache hit for track {track_id} lacks album data, fetching full data") - else: + elif allow_fallback: # Fallback cache hit — delegate to fallback client which reconstructs enhanced format return self._fallback.get_track_details(track_id) @@ -1298,7 +1320,7 @@ class SpotifyClient: # Fall through to iTunes fallback # Fallback - only if ID is numeric (non-Spotify format) - if self._is_itunes_id(track_id): + if allow_fallback and self._is_itunes_id(track_id): logger.debug(f"Using {fallback_src} fallback for track details: {track_id}") result = self._fallback.get_track_details(track_id) return result @@ -1354,8 +1376,12 @@ class SpotifyClient: return None @rate_limited - def get_album(self, album_id: str) -> Optional[Dict[str, Any]]: - """Get album information - falls back to configured metadata source""" + def get_album(self, album_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]: + """Get album information. + + When allow_fallback is True, falls back to the configured metadata source + for non-Spotify IDs or Spotify failure. + """ # Check cache first cache = get_metadata_cache() fallback_src = self._fallback_source @@ -1368,7 +1394,7 @@ class SpotifyClient: return cached # Simplified album cached by get_artist_albums — treat as cache miss logger.debug(f"Cache hit for album {album_id} lacks tracks, fetching full data") - else: + elif allow_fallback: # Fallback cache hit — delegate to fallback client return self._fallback.get_album(album_id) @@ -1385,7 +1411,7 @@ class SpotifyClient: # Fall through to fallback # Fallback - only if ID is numeric (non-Spotify format) - if self._is_itunes_id(album_id): + if allow_fallback and self._is_itunes_id(album_id): logger.debug(f"Using {fallback_src} fallback for album: {album_id}") return self._fallback.get_album(album_id) else: @@ -1393,8 +1419,12 @@ class SpotifyClient: return None @rate_limited - def get_album_tracks(self, album_id: str) -> Optional[Dict[str, Any]]: - """Get album tracks - falls back to configured metadata source""" + def get_album_tracks(self, album_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]: + """Get album tracks. + + When allow_fallback is True, falls back to the configured metadata source + for non-Spotify IDs or Spotify failure. + """ # Cache key uses album_id with '_tracks' suffix to differentiate from album metadata cache = get_metadata_cache() fallback_src = self._fallback_source @@ -1458,7 +1488,7 @@ class SpotifyClient: # Fall through to iTunes fallback # Fallback - only if ID is numeric (non-Spotify format) - if self._is_itunes_id(album_id): + if allow_fallback and self._is_itunes_id(album_id): logger.debug(f"Using {fallback_src} fallback for album tracks: {album_id}") result = self._fallback.get_album_tracks(album_id) return result @@ -1467,8 +1497,12 @@ class SpotifyClient: return None @rate_limited - def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 10, skip_cache: bool = False, max_pages: int = 0) -> List[Album]: - """Get albums by artist ID - falls back to iTunes if Spotify not authenticated. + def get_artist_albums(self, artist_id: str, album_type: str = 'album,single', limit: int = 10, + skip_cache: bool = False, max_pages: int = 0, allow_fallback: bool = True) -> List[Album]: + """Get albums by artist ID. + + When allow_fallback is True, falls back to iTunes/Deezer if Spotify + is not authenticated or errors. Set skip_cache=True for watchlist scans that need fresh data to detect new releases. Set max_pages to limit pagination (0 = fetch all). Spotify returns newest first, so max_pages=1 is sufficient for new release detection.""" @@ -1540,7 +1574,7 @@ class SpotifyClient: # Fall through to iTunes fallback # Fallback - only if ID is numeric (non-Spotify format) - if self._is_itunes_id(artist_id): + if allow_fallback and self._is_itunes_id(artist_id): logger.debug(f"Using {fallback_src} fallback for artist albums: {artist_id}") return self._fallback.get_artist_albums(artist_id, album_type, limit) else: @@ -1559,9 +1593,9 @@ class SpotifyClient: return None @rate_limited - def get_artist(self, artist_id: str) -> Optional[Dict[str, Any]]: + def get_artist(self, artist_id: str, allow_fallback: bool = True) -> Optional[Dict[str, Any]]: """ - Get full artist details - falls back to configured metadata source. + Get full artist details. Args: artist_id: Artist ID (Spotify or fallback source depending on authentication) @@ -1577,8 +1611,10 @@ class SpotifyClient: if cached: if source == 'spotify': return cached # Spotify raw format is the expected format - # Fallback cache hit — delegate to fallback client which reconstructs Spotify-compatible format - return self._fallback.get_artist(artist_id) + if allow_fallback: + # Fallback cache hit — delegate to fallback client which reconstructs Spotify-compatible format + return self._fallback.get_artist(artist_id) + return None if self.is_spotify_authenticated(): try: @@ -1592,7 +1628,7 @@ class SpotifyClient: # Fall through to iTunes fallback # Fallback - only if ID is numeric (non-Spotify format) - if self._is_itunes_id(artist_id): + if allow_fallback and self._is_itunes_id(artist_id): logger.debug(f"Using {fallback_src} fallback for artist: {artist_id}") return self._fallback.get_artist(artist_id) else: From 992be1a0560a198deb711e77eb081f1832ad8a87 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Thu, 16 Apr 2026 09:54:00 +0300 Subject: [PATCH 08/10] Use cache before Spotify auth checks Allow cached Spotify search results to return even when Spotify is rate-limited or temporarily unavailable, and remove redundant rate-limit gating after auth checks. --- core/spotify_client.py | 166 +++++++++++++++++++---------------------- 1 file changed, 77 insertions(+), 89 deletions(-) diff --git a/core/spotify_client.py b/core/spotify_client.py index 60d68232..0e2c962e 100644 --- a/core/spotify_client.py +++ b/core/spotify_client.py @@ -1105,49 +1105,46 @@ class SpotifyClient: if Spotify is unavailable or returns an error. """ cache = get_metadata_cache() + effective_limit = min(limit, 50) # Spotify API max is 50 + + # Check Spotify cache first so cached data remains usable even when + # Spotify is temporarily unavailable or rate limited. + cached_results = cache.get_search_results('spotify', 'track', query, effective_limit) + if cached_results is not None: + tracks = [] + for raw in cached_results: + try: + tracks.append(Track.from_spotify_track(raw)) + except Exception: + pass + if tracks: + return tracks + use_spotify = self.is_spotify_authenticated() if use_spotify: - # Check Spotify cache - effective_limit = min(limit, 50) # Spotify API max is 50 - cached_results = cache.get_search_results('spotify', 'track', query, effective_limit) - if cached_results is not None: + try: + results = self.sp.search(q=query, type='track', limit=effective_limit) tracks = [] - for raw in cached_results: - try: - tracks.append(Track.from_spotify_track(raw)) - except Exception: - pass - if tracks: - return tracks + raw_items = results['tracks']['items'] - # Skip Spotify if globally rate limited — fall through to fallback - if self.is_rate_limited(): - logger.debug(f"Spotify rate limited, skipping track search for: {query}") - use_spotify = False - else: - try: - results = self.sp.search(q=query, type='track', limit=effective_limit) - tracks = [] - raw_items = results['tracks']['items'] + for track_data in raw_items: + track = Track.from_spotify_track(track_data) + tracks.append(track) - for track_data in raw_items: - track = Track.from_spotify_track(track_data) - tracks.append(track) + # Cache individual tracks + search mapping + entries = [(td.get('id'), td) for td in raw_items if td.get('id')] + if entries: + cache.store_entities_bulk('spotify', 'track', entries) + cache.store_search_results('spotify', 'track', query, effective_limit, + [td.get('id') for td in raw_items if td.get('id')]) - # Cache individual tracks + search mapping - entries = [(td.get('id'), td) for td in raw_items if td.get('id')] - if entries: - cache.store_entities_bulk('spotify', 'track', entries) - cache.store_search_results('spotify', 'track', query, effective_limit, - [td.get('id') for td in raw_items if td.get('id')]) + return tracks - return tracks - - except Exception as e: - _detect_and_set_rate_limit(e, 'search_tracks') - logger.error(f"Error searching tracks via Spotify: {e}") - # Fall through to fallback + except Exception as e: + _detect_and_set_rate_limit(e, 'search_tracks') + logger.error(f"Error searching tracks via Spotify: {e}") + # Fall through to fallback # Fallback (iTunes or Deezer — configured in settings) if allow_fallback: @@ -1163,27 +1160,23 @@ class SpotifyClient: if Spotify is unavailable or returns an error. """ cache = get_metadata_cache() + # Check Spotify cache first so cached data remains usable even when + # Spotify is temporarily unavailable or rate limited. + cached_results = cache.get_search_results('spotify', 'artist', query, min(limit, 10)) + if cached_results is not None: + artists = [] + for raw in cached_results: + try: + artists.append(Artist.from_spotify_artist(raw)) + except Exception: + pass + if artists: + query_lower = query.lower().strip() + artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1)) + return artists + use_spotify = self.is_spotify_authenticated() - if use_spotify: - # Check Spotify cache - cached_results = cache.get_search_results('spotify', 'artist', query, min(limit, 10)) - if cached_results is not None: - artists = [] - for raw in cached_results: - try: - artists.append(Artist.from_spotify_artist(raw)) - except Exception: - pass - if artists: - query_lower = query.lower().strip() - artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1)) - return artists - - if self.is_rate_limited(): - logger.debug(f"Spotify rate limited, skipping artist search for: {query}") - use_spotify = False - if use_spotify: try: search_query = f'artist:{query}' if len(query.strip()) <= 4 else query @@ -1230,49 +1223,44 @@ class SpotifyClient: if Spotify is unavailable or returns an error. """ cache = get_metadata_cache() + # Check Spotify cache first so cached data remains usable even when + # Spotify is temporarily unavailable or rate limited. + cached_results = cache.get_search_results('spotify', 'album', query, min(limit, 10)) + if cached_results is not None: + albums = [] + for raw in cached_results: + try: + albums.append(Album.from_spotify_album(raw)) + except Exception: + pass + if albums: + return albums + use_spotify = self.is_spotify_authenticated() if use_spotify: - # Check Spotify cache - cached_results = cache.get_search_results('spotify', 'album', query, min(limit, 10)) - if cached_results is not None: + try: + results = self.sp.search(q=query, type='album', limit=min(limit, 10)) albums = [] - for raw in cached_results: - try: - albums.append(Album.from_spotify_album(raw)) - except Exception: - pass - if albums: - return albums + raw_items = results['albums']['items'] - if use_spotify: - # Skip Spotify if globally rate limited — fall through to fallback - if self.is_rate_limited(): - logger.debug(f"Spotify rate limited, skipping album search for: {query}") - use_spotify = False - else: - try: - results = self.sp.search(q=query, type='album', limit=min(limit, 10)) - albums = [] - raw_items = results['albums']['items'] + for album_data in raw_items: + album = Album.from_spotify_album(album_data) + albums.append(album) - for album_data in raw_items: - album = Album.from_spotify_album(album_data) - albums.append(album) + # Cache individual albums + search mapping (skip if full data already cached) + entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')] + if entries: + cache.store_entities_bulk('spotify', 'album', entries, skip_if_exists=True) + cache.store_search_results('spotify', 'album', query, min(limit, 10), + [ad.get('id') for ad in raw_items if ad.get('id')]) - # Cache individual albums + search mapping (skip if full data already cached) - entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')] - if entries: - cache.store_entities_bulk('spotify', 'album', entries, skip_if_exists=True) - cache.store_search_results('spotify', 'album', query, min(limit, 10), - [ad.get('id') for ad in raw_items if ad.get('id')]) + return albums - return albums - - except Exception as e: - _detect_and_set_rate_limit(e, 'search_albums') - logger.error(f"Error searching albums via Spotify: {e}") - # Fall through to iTunes fallback + except Exception as e: + _detect_and_set_rate_limit(e, 'search_albums') + logger.error(f"Error searching albums via Spotify: {e}") + # Fall through to iTunes fallback # Fallback (iTunes or Deezer) if allow_fallback: From e657a1d4322cb4d9c0ba2aec01ccef6a42b566d3 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Thu, 16 Apr 2026 09:59:49 +0300 Subject: [PATCH 09/10] Make watchlist Spotify matching strict Resolve Spotify artist matching through the exact Spotify client only, so watchlist ID backfill cannot drift to fallback-provider results. Remove the remaining preemptive provider availability check from the backfill loop. --- core/watchlist_scanner.py | 63 +++++++++++++++------------- tests/test_watchlist_scanner_scan.py | 33 +++++++++++++-- 2 files changed, 64 insertions(+), 32 deletions(-) diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 32399b71..3d6e7553 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -355,7 +355,6 @@ class ScanResult: class WatchlistDiscographyResult: """Resolved watchlist artist discography for a specific metadata source.""" source: str - client: Any artist_id: str albums: List[Any] image_url: Optional[str] = None @@ -560,8 +559,9 @@ class WatchlistScanner: return self._get_artist_image_from_data(artist_data) - def _get_album_data_for_source(self, source: str, client: Any, album_id: str, album_name: str = '') -> Optional[Dict[str, Any]]: + def _get_album_data_for_source(self, source: str, album_id: str, album_name: str = '') -> Optional[Dict[str, Any]]: """Fetch album data for a specific source and normalize track payloads when needed.""" + client = get_client_for_source(source) if not client or not album_id or not hasattr(client, 'get_album'): return None @@ -642,7 +642,6 @@ class WatchlistScanner: image_url = self._get_artist_image_for_source(watchlist_artist, source, client, artist_id) return WatchlistDiscographyResult( source=source, - client=client, artist_id=artist_id, albums=albums, image_url=image_url, @@ -913,14 +912,11 @@ class WatchlistScanner: _emit('scan_started', profile_id=profile_id, total_artists=len(watchlist_artists)) - # Rate-limit-aware backfill of artist IDs for the providers we can safely resolve. - providers_to_backfill = [] - for source in self._watchlist_source_priority(): - if source == 'spotify': - if self._spotify_available_for_run(): - providers_to_backfill.append(source) - elif source in {'itunes', 'deezer', 'discogs'} and get_client_for_source(source): - providers_to_backfill.append(source) + # Keep this as a plain source list; resolve the client right before each use. + providers_to_backfill = [ + source for source in self._watchlist_source_priority() + if source in {'spotify', 'itunes', 'deezer', 'discogs'} + ] for provider in providers_to_backfill: try: @@ -1005,11 +1001,10 @@ class WatchlistScanner: album_fetcher = lambda album_id, album_name='': self.metadata_service.get_album(album_id) else: source = discography_result.source - client = discography_result.client albums = discography_result.albums source_artist_id = discography_result.artist_id artist_image_url = discography_result.image_url or self.get_artist_image_url(artist) or '' - album_fetcher = lambda album_id, album_name='': self._get_album_data_for_source(source, client, album_id, album_name) + album_fetcher = lambda album_id, album_name='': self._get_album_data_for_source(source, album_id, album_name) absolute_index = artist_index_offset + i + 1 if scan_state is not None: @@ -1251,8 +1246,11 @@ class WatchlistScanner: logger.debug(f"Fetching discography for artist {spotify_artist_id}" + (" (full)" if needs_full_discog else " (recent only, max 1 page)")) albums = self.spotify_client.get_artist_albums( - spotify_artist_id, album_type='album,single', limit=50, - skip_cache=True, max_pages=0 if needs_full_discog else 1 + spotify_artist_id, + album_type='album,single', + limit=50, + skip_cache=True, + max_pages=0 if needs_full_discog else 1, ) if not albums: @@ -1413,8 +1411,8 @@ class WatchlistScanner: update_fn = { 'spotify': self.database.update_watchlist_spotify_id, 'itunes': self.database.update_watchlist_itunes_id, - 'deezer': getattr(self.database, 'update_watchlist_deezer_id', None), - 'discogs': getattr(self.database, 'update_watchlist_discogs_id', None), + 'deezer': self.database.update_watchlist_deezer_id, + 'discogs': self.database.update_watchlist_discogs_id, }.get(provider) if not match_fn or not update_fn: @@ -1518,10 +1516,11 @@ class WatchlistScanner: def _match_to_spotify(self, artist_name: str) -> Optional[str]: """Match artist name to Spotify ID using fuzzy name comparison.""" try: - if hasattr(self, '_metadata_service') and self._metadata_service: - results = self._metadata_service.spotify.search_artists(artist_name, limit=5) - else: - results = self.spotify_client.search_artists(artist_name, limit=5) + client = get_client_for_source('spotify') + if not client: + return None + + results = client.search_artists(artist_name, limit=5, allow_fallback=False) return self._best_artist_match(results, artist_name) except Exception as e: @@ -2101,7 +2100,7 @@ class WatchlistScanner: try: # Try Spotify search (only when Spotify is the configured primary source) if self._spotify_is_primary_source(): - searched_results = self.spotify_client.search_artists(artist_name, limit=1) + searched_results = self.spotify_client.search_artists(artist_name, limit=1, allow_fallback=False) if searched_results and len(searched_results) > 0: searched_spotify_id = searched_results[0].id except Exception as e: @@ -2139,7 +2138,11 @@ class WatchlistScanner: # Try to match on Spotify (only when Spotify is the configured primary source) if self._spotify_is_primary_source(): try: - spotify_results = self.spotify_client.search_artists(artist_name_to_match, limit=1) + spotify_results = self.spotify_client.search_artists( + artist_name_to_match, + limit=1, + allow_fallback=False, + ) if spotify_results and len(spotify_results) > 0: spotify_artist = spotify_results[0] # Skip if this is the searched artist @@ -2443,7 +2446,7 @@ class WatchlistScanner: artist_id, album_type='album,single,ep', limit=50, - skip_cache=True + skip_cache=True, ) else: # itunes or deezer fallback all_albums = itunes_client.get_artist_albums( @@ -2668,7 +2671,11 @@ class WatchlistScanner: # Try Spotify first if available if spotify_available: try: - search_results = self.spotify_client.search_albums(f"album:{album_row['title']} artist:{album_row['artist_name']}", limit=1) + search_results = self.spotify_client.search_albums( + f"album:{album_row['title']} artist:{album_row['artist_name']}", + limit=1, + allow_fallback=False, + ) if search_results and len(search_results) > 0: spotify_album = search_results[0] album_data = self.spotify_client.get_album(spotify_album.id) @@ -2854,7 +2861,7 @@ class WatchlistScanner: artist.spotify_artist_id, album_type='album,single,ep', limit=5, - skip_cache=True + skip_cache=True, ) if not recent_releases: @@ -3095,7 +3102,7 @@ class WatchlistScanner: artist.spotify_artist_id, album_type='album,single,ep', limit=20, - skip_cache=True + skip_cache=True, ) for album in albums or []: process_album(album, artist.artist_name, artist.spotify_artist_id, fallback_id if fallback_source == 'itunes' else None, 'spotify') @@ -3152,7 +3159,7 @@ class WatchlistScanner: artist.similar_artist_spotify_id, album_type='album,single,ep', limit=20, - skip_cache=True + skip_cache=True, ) for album in albums or []: process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, fallback_id if fallback_source == 'itunes' else None, 'spotify') diff --git a/tests/test_watchlist_scanner_scan.py b/tests/test_watchlist_scanner_scan.py index 6b562b22..57d8909d 100644 --- a/tests/test_watchlist_scanner_scan.py +++ b/tests/test_watchlist_scanner_scan.py @@ -53,13 +53,21 @@ from core.watchlist_scanner import WatchlistScanner class _FakeSpotifyClient: + def __init__(self, search_results=None): + self.search_results = list(search_results or []) + self.search_calls = [] + def is_spotify_authenticated(self): return False + def search_artists(self, query, limit=1, allow_fallback=True): + self.search_calls.append((query, limit, allow_fallback)) + return list(self.search_results) if allow_fallback else [] + class _FakeMetadataService: - def __init__(self, album_data): - self.spotify = _FakeSpotifyClient() + def __init__(self, album_data, spotify_client=None): + self.spotify = spotify_client or _FakeSpotifyClient() self.itunes = types.SimpleNamespace() self._album_data = album_data @@ -76,8 +84,8 @@ class _FakeSourceClient: self.album_calls = [] self.artist_calls = [] - def search_artists(self, query, limit=1): - self.search_calls.append((query, limit)) + def search_artists(self, query, limit=1, **kwargs): + self.search_calls.append((query, limit, kwargs)) return [types.SimpleNamespace(id=self.artist_id, name=query)] def get_artist_albums(self, artist_id, album_type='album,single', limit=50, **kwargs): @@ -374,3 +382,20 @@ def test_get_artist_discography_for_watchlist_falls_back_when_primary_empty(monk assert result.albums and result.albums[0].id == "sp-album" assert deezer_client.album_calls assert spotify_client.album_calls + + +def test_match_to_spotify_uses_strict_lookup(): + spotify_client = _FakeSpotifyClient( + search_results=[types.SimpleNamespace(id="fallback-id", name="Artist One")] + ) + scanner = WatchlistScanner(metadata_service=_FakeMetadataService(None, spotify_client=spotify_client)) + original_get_client_for_source = watchlist_scanner_module.get_client_for_source + watchlist_scanner_module.get_client_for_source = lambda source: spotify_client if source == "spotify" else None + + try: + result = scanner._match_to_spotify("Artist One") + finally: + watchlist_scanner_module.get_client_for_source = original_get_client_for_source + + assert result is None + assert spotify_client.search_calls == [("Artist One", 5, False)] From 08ac39bc131aefe8a6f9408b4407fb2c8f0f9021 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Thu, 16 Apr 2026 10:09:42 +0300 Subject: [PATCH 10/10] Fix watchlist discography lookback handling Route get_artist_discography through the shared client helper so it uses the existing lookback logic instead of referencing an out-of-scope variable. --- core/watchlist_scanner.py | 65 +++++++-------------------------------- 1 file changed, 11 insertions(+), 54 deletions(-) diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 3d6e7553..643d9b93 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -1212,7 +1212,12 @@ class WatchlistScanner: ) return scan_results - def get_artist_discography(self, spotify_artist_id: str, last_scan_timestamp: Optional[datetime] = None) -> Optional[List]: + def get_artist_discography( + self, + spotify_artist_id: str, + last_scan_timestamp: Optional[datetime] = None, + lookback_days: Optional[int] = None, + ) -> Optional[List]: """ Get artist's discography from Spotify, optionally filtered by release date. @@ -1220,64 +1225,16 @@ class WatchlistScanner: spotify_artist_id: Spotify artist ID last_scan_timestamp: Only return releases after this date (for incremental scans) If None, uses lookback period setting from database + lookback_days: Optional per-artist override for lookback period """ try: - # Determine if we need the full discography or just recent releases. - # Spotify returns albums sorted newest-first, so for time-bounded scans - # we only need the first page (50 albums) — this cuts API calls by ~90% - # for prolific artists (262 albums = 27 calls → 1 call). - needs_full_discog = False - cutoff_timestamp = last_scan_timestamp - - if cutoff_timestamp is None: - if lookback_days is not None: - cutoff_timestamp = datetime.now(timezone.utc) - timedelta(days=lookback_days) - logger.info(f"Using per-artist lookback: {lookback_days} days (cutoff: {cutoff_timestamp})") - else: - lookback_period = self._get_lookback_period_setting() - if lookback_period == 'all': - needs_full_discog = True - else: - days = int(lookback_period) - cutoff_timestamp = datetime.now(timezone.utc) - timedelta(days=days) - logger.info(f"Using global lookback period: {lookback_period} days (cutoff: {cutoff_timestamp})") - - # Fetch albums — limit pagination unless full discography is needed - logger.debug(f"Fetching discography for artist {spotify_artist_id}" + - (" (full)" if needs_full_discog else " (recent only, max 1 page)")) - albums = self.spotify_client.get_artist_albums( + return self._get_artist_discography_with_client( + self.spotify_client, spotify_artist_id, - album_type='album,single', - limit=50, - skip_cache=True, - max_pages=0 if needs_full_discog else 1, + last_scan_timestamp, + lookback_days=lookback_days, ) - if not albums: - logger.warning(f"No albums found for artist {spotify_artist_id}") - return [] - - # Add small delay after fetching artist discography to be extra safe - time.sleep(0.3) # 300ms breathing room - - # Filter by release date if we have a cutoff timestamp - if cutoff_timestamp: - filtered_albums = [] - for album in albums: - if self.is_album_after_timestamp(album, cutoff_timestamp): - filtered_albums.append(album) - - logger.info(f"Filtered {len(albums)} albums to {len(filtered_albums)} released after {cutoff_timestamp}") - albums = filtered_albums - - # Skip future/unreleased albums — no real audio available yet - now = datetime.now(timezone.utc) - released = [a for a in albums if not self._is_future_release(a, now)] - skipped = len(albums) - len(released) - if skipped: - logger.info(f"Skipped {skipped} future/unreleased albums (will be picked up after release)") - return released - except Exception as e: logger.error(f"Error getting discography for artist {spotify_artist_id}: {e}") return None