danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Reduce discovery fan-out and pagination

Browse source 
Make discovery pool population respect provider priority while keeping Spotify strict, and reduce unnecessary request volume in the hot discovery paths.

- keep discovery fan-out source-priority aware
- preserve cache use where freshness is not required
- cap Spotify artist-album pagination in discovery and cache refresh paths
- keep incremental release checks to a single page, since they only need the newest releases
- add regression coverage for provider order, strict Spotify handling, and pagination caps
This commit is contained in:
Antti Kettunen 2026-04-16 20:59:26 +03:00
parent 0b60986f44
commit e447cf6ab0
3 changed files with 637 additions and 283 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
core/metadata_service.py
View file

@ -128,6 +128,8 @@ def get_album_tracks_for_source(source: str, album_id: str):
fetch = getattr(client, 'get_album_tracks_dict', None) if source == 'hydrabase' else getattr(client, 'get_album_tracks', None) fetch = getattr(client, 'get_album_tracks_dict', None) if source == 'hydrabase' else getattr(client, 'get_album_tracks', None)
if not fetch: if not fetch:
return None return None
if source == 'spotify':
return fetch(album_id, allow_fallback=False)
return fetch(album_id) return fetch(album_id)
except Exception: except Exception:
return None return None

363
core/watchlist_scanner.py
View file

@ -451,6 +451,14 @@ class WatchlistScanner:
"""Return watchlist scan sources in the configured priority order.""" """Return watchlist scan sources in the configured priority order."""
return list(get_source_priority(get_primary_source())) return list(get_source_priority(get_primary_source()))
def _discovery_source_priority(self) -> List[str]:
"""Return discovery sources in configured priority order.
Discovery pool writes only support Spotify, iTunes, and Deezer IDs, so
we filter the broader metadata priority list down to those sources.
"""
return [source for source in self._watchlist_source_priority() if source in {'spotify', 'itunes', 'deezer'}]
@staticmethod @staticmethod
def _artist_id_attribute_for_source(source: str) -> Optional[str]: def _artist_id_attribute_for_source(source: str) -> Optional[str]:
"""Return the watchlist artist attribute that stores the given source ID.""" """Return the watchlist artist attribute that stores the given source ID."""
@ -501,7 +509,10 @@ class WatchlistScanner:
return None return None
try: try:
search_results = client.search_artists(watchlist_artist.artist_name, limit=1) search_kwargs = {'limit': 1}
if source == 'spotify':
search_kwargs['allow_fallback'] = False
search_results = client.search_artists(watchlist_artist.artist_name, **search_kwargs)
except Exception as e: except Exception as e:
logger.debug("Could not search %s for %s: %s", source, watchlist_artist.artist_name, e) logger.debug("Could not search %s for %s: %s", source, watchlist_artist.artist_name, e)
return None return None
@ -552,6 +563,9 @@ class WatchlistScanner:
return None return None
try: try:
if source == 'spotify':
artist_data = client.get_artist(artist_id, allow_fallback=False)
else:
artist_data = client.get_artist(artist_id) artist_data = client.get_artist(artist_id)
except Exception as e: except Exception as e:
logger.debug("Could not fetch artist image for %s on %s: %s", watchlist_artist.artist_name, source, e) logger.debug("Could not fetch artist image for %s on %s: %s", watchlist_artist.artist_name, source, e)
@ -566,6 +580,9 @@ class WatchlistScanner:
return None return None
try: try:
if source == 'spotify':
album_data = client.get_album(album_id, allow_fallback=False)
else:
album_data = client.get_album(album_id) album_data = client.get_album(album_id)
except Exception as e: except Exception as e:
logger.debug("Could not fetch album %s on %s: %s", album_id, source, e) logger.debug("Could not fetch album %s on %s: %s", album_id, source, e)
@ -666,6 +683,97 @@ class WatchlistScanner:
return image_url return image_url
return None return None
def _get_artist_albums_for_source(
self,
source: str,
artist_id: str,
album_type: str = 'album,single,ep',
limit: int = 50,
skip_cache: bool = True,
max_pages: int = 0,
) -> List[Any]:
"""Fetch artist albums for a specific source, keeping Spotify strict."""
client = get_client_for_source(source)
if not client or not artist_id or not hasattr(client, 'get_artist_albums'):
return []
try:
kwargs = {
'album_type': album_type,
'limit': limit,
}
if source == 'spotify':
kwargs['skip_cache'] = skip_cache
kwargs['max_pages'] = max_pages
kwargs['allow_fallback'] = False
return client.get_artist_albums(artist_id, **kwargs) or []
except Exception as e:
logger.debug("Could not fetch artist albums for %s on %s: %s", artist_id, source, e)
return []
def _get_artist_data_for_source(self, source: str, artist_id: str) -> Optional[Dict[str, Any]]:
"""Fetch artist metadata for a specific source, keeping Spotify strict."""
client = get_client_for_source(source)
if not client or not artist_id or not hasattr(client, 'get_artist'):
return None
try:
if source == 'spotify':
return client.get_artist(artist_id, allow_fallback=False)
return client.get_artist(artist_id)
except Exception as e:
logger.debug("Could not fetch artist data for %s on %s: %s", artist_id, source, e)
return None
def _search_albums_for_source(self, source: str, query: str, limit: int = 1):
"""Search albums for a specific source, keeping Spotify strict."""
client = get_client_for_source(source)
if not client or not hasattr(client, 'search_albums'):
return []
try:
if source == 'spotify':
return client.search_albums(query, limit=limit, allow_fallback=False) or []
return client.search_albums(query, limit=limit) or []
except Exception as e:
logger.debug("Could not search albums for %s on %s: %s", query, source, e)
return []
def _resolve_artist_id_for_source(
self,
source: str,
artist_name: str,
stored_id: Optional[str] = None,
cache_callback: Optional[Callable[[str], None]] = None,
) -> Optional[str]:
"""Resolve an artist ID for a specific source, searching by name if needed."""
if stored_id:
return stored_id
client = get_client_for_source(source)
if not client or not hasattr(client, 'search_artists'):
return None
try:
search_kwargs = {'limit': 1}
if source == 'spotify':
search_kwargs['allow_fallback'] = False
results = client.search_artists(artist_name, **search_kwargs)
except Exception as e:
logger.debug("Could not resolve %s artist ID for %s: %s", source, artist_name, e)
return None
if not results:
return None
found_id = self._extract_entity_id(results[0])
if found_id and cache_callback:
try:
cache_callback(found_id)
except Exception as e:
logger.debug("Could not cache %s artist ID for %s: %s", source, artist_name, e)
return found_id
def backfill_watchlist_artist_images(self, profile_id: int) -> int: def backfill_watchlist_artist_images(self, profile_id: int) -> int:
"""Backfill missing watchlist artist images using cached metadata and existing album art.""" """Backfill missing watchlist artist images using cached metadata and existing album art."""
try: try:
@ -2298,9 +2406,6 @@ class WatchlistScanner:
from datetime import datetime, timedelta from datetime import datetime, timedelta
import random import random
if self.spotify_client and self.spotify_client.is_rate_limited():
self._disable_spotify_for_run("global Spotify rate limit active")
# Check if we should run discovery pool population (prevents over-polling) # Check if we should run discovery pool population (prevents over-polling)
skip_pool_population = not self.database.should_populate_discovery_pool(hours_threshold=24, profile_id=profile_id) skip_pool_population = not self.database.should_populate_discovery_pool(hours_threshold=24, profile_id=profile_id)
@ -2320,18 +2425,12 @@ class WatchlistScanner:
logger.info("Populating discovery pool from similar artists...") logger.info("Populating discovery pool from similar artists...")
# Determine which sources are available discovery_sources = self._discovery_source_priority()
spotify_available = self._spotify_is_primary_source() if not discovery_sources:
# Import fallback metadata client (iTunes or Deezer)
itunes_client, fallback_source = _get_fallback_metadata_client()
fallback_available = True # Fallback source is always available (no auth needed)
if not spotify_available and not fallback_available:
logger.warning("No music sources available to populate discovery pool") logger.warning("No music sources available to populate discovery pool")
return return
logger.info(f"Sources available - Spotify: {spotify_available}, {fallback_source}: {fallback_available}") logger.info("Discovery source priority: %s", discovery_sources)
# Get top similar artists for this profile's watchlist (ordered by occurrence_count) # Get top similar artists for this profile's watchlist (ordered by occurrence_count)
similar_artists = self.database.get_top_similar_artists(limit=top_artists_limit, profile_id=profile_id) similar_artists = self.database.get_top_similar_artists(limit=top_artists_limit, profile_id=profile_id)
@ -2360,146 +2459,99 @@ class WatchlistScanner:
if progress_callback: if progress_callback:
progress_callback('artist', f'{similar_artist.similar_artist_name} ({artist_idx}/{len(similar_artists)})') progress_callback('artist', f'{similar_artist.similar_artist_name} ({artist_idx}/{len(similar_artists)})')
# Build list of sources to process for this artist # Resolve the first source that can actually produce albums.
# Fallback source (iTunes/Deezer) is ALWAYS processed (baseline), Spotify is added if authenticated selected_source = None
sources_to_process = [] selected_artist_id = None
selected_albums = []
artist_genres: List[str] = []
# Always add fallback source first (baseline source) for source in discovery_sources:
fallback_id = similar_artist.similar_artist_itunes_id if fallback_source == 'itunes' else getattr(similar_artist, 'similar_artist_deezer_id', None) source_attr = self._artist_id_attribute_for_source(source)
if not fallback_id: stored_id = getattr(similar_artist, source_attr, None) if source_attr else None
# On-the-fly lookup for missing fallback ID (seamless provider switching)
try:
fallback_results = itunes_client.search_artists(similar_artist.similar_artist_name, limit=1)
if fallback_results and len(fallback_results) > 0:
fallback_id = fallback_results[0].id
# Cache it for future use
if fallback_source == 'deezer':
self.database.update_similar_artist_deezer_id(similar_artist.id, fallback_id)
else:
self.database.update_similar_artist_itunes_id(similar_artist.id, fallback_id)
logger.debug(f" Resolved {fallback_source} ID {fallback_id} for {similar_artist.similar_artist_name}")
except Exception as e:
logger.debug(f" Could not resolve {fallback_source} ID for {similar_artist.similar_artist_name}: {e}")
if fallback_id: cache_callback = None
sources_to_process.append((fallback_source, fallback_id)) if source == 'itunes':
cache_callback = lambda found_id, artist_id=similar_artist.id: self.database.update_similar_artist_itunes_id(artist_id, found_id)
elif source == 'deezer':
cache_callback = lambda found_id, artist_id=similar_artist.id: self.database.update_similar_artist_deezer_id(artist_id, found_id)
# Add Spotify if authenticated and we have an ID artist_id = self._resolve_artist_id_for_source(
if spotify_available and similar_artist.similar_artist_spotify_id: source,
sources_to_process.append(('spotify', similar_artist.similar_artist_spotify_id)) similar_artist.similar_artist_name,
stored_id=stored_id,
if not sources_to_process: cache_callback=cache_callback,
logger.debug(f"No valid IDs for {similar_artist.similar_artist_name}, skipping") )
if not artist_id:
continue continue
logger.debug(f" Processing {len(sources_to_process)} source(s): {[s[0] for s in sources_to_process]}") all_albums = self._get_artist_albums_for_source(
source,
# Process each source for this artist
for source, artist_id in sources_to_process:
try:
# Get artist's albums from this source
if source == 'spotify':
all_albums = self.spotify_client.get_artist_albums(
artist_id, artist_id,
album_type='album,single,ep', album_type='album,single,ep',
limit=50, limit=50,
skip_cache=True, skip_cache=False,
max_pages=2,
) )
else: # itunes or deezer fallback
all_albums = itunes_client.get_artist_albums(
artist_id,
album_type='album,single,ep',
limit=50
)
if not all_albums: if not all_albums:
logger.debug(f"No albums found for {similar_artist.similar_artist_name} on {source}") logger.debug(f"No albums found for {similar_artist.similar_artist_name} on {source}")
continue continue
# Fetch artist genres for this source artist_data = self._get_artist_data_for_source(source, artist_id)
artist_genres = []
try:
if source == 'spotify':
artist_data = self.spotify_client.get_artist(artist_id)
if artist_data and 'genres' in artist_data: if artist_data and 'genres' in artist_data:
artist_genres = artist_data['genres'] artist_genres = artist_data['genres']
else: # itunes/deezer - genres from artist lookup
artist_data = itunes_client.get_artist(artist_id)
if artist_data and 'genres' in artist_data:
artist_genres = artist_data['genres']
except Exception as e:
logger.debug(f"Could not fetch genres for {similar_artist.similar_artist_name} on {source}: {e}")
# IMPROVED: Smart selection mixing albums, singles, and EPs
# Prioritize recent releases and popular content
# Separate by type for balanced selection
albums = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type == 'album'] albums = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type == 'album']
singles_eps = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type in ['single', 'ep']] singles_eps = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type in ['single', 'ep']]
other = [a for a in all_albums if not hasattr(a, 'album_type')]
# Select albums: latest releases + popular older content
selected_albums = [] selected_albums = []
# Always include 3 most recent releases (any type) - this captures new singles/EPs
latest_releases = all_albums[:3] latest_releases = all_albums[:3]
selected_albums.extend(latest_releases) selected_albums.extend(latest_releases)
# Add remaining slots with balanced mix
remaining_slots = albums_per_artist - len(selected_albums) remaining_slots = albums_per_artist - len(selected_albums)
if remaining_slots > 0: if remaining_slots > 0:
# Combine remaining albums and singles
remaining_content = all_albums[3:] remaining_content = all_albums[3:]
if len(remaining_content) > remaining_slots: if len(remaining_content) > remaining_slots:
# Randomly select from remaining content selected_albums.extend(random.sample(remaining_content, remaining_slots))
random_selection = random.sample(remaining_content, remaining_slots)
selected_albums.extend(random_selection)
else: else:
selected_albums.extend(remaining_content) selected_albums.extend(remaining_content)
logger.info(f" [{source}] Selected {len(selected_albums)} releases from {len(all_albums)} available (albums: {len(albums)}, singles/EPs: {len(singles_eps)})") selected_source = source
selected_artist_id = artist_id
logger.info(
f" [{source}] Selected {len(selected_albums)} releases from {len(all_albums)} available "
f"(albums: {len(albums)}, singles/EPs: {len(singles_eps)})"
)
break
# Process each selected album if not selected_source or not selected_artist_id or not selected_albums:
logger.debug(f"No valid source/albums for {similar_artist.similar_artist_name}, skipping")
continue
# Process each selected album from the winning source.
for album_idx, album in enumerate(selected_albums, 1): for album_idx, album in enumerate(selected_albums, 1):
try: try:
# Get full album data with tracks from appropriate source album_data = self._get_album_data_for_source(selected_source, album.id, album_name=album.name)
if source == 'spotify':
album_data = self.spotify_client.get_album(album.id)
if not album_data or 'tracks' not in album_data:
continue
tracks = album_data['tracks'].get('items', [])
else: # itunes or deezer fallback
album_data = itunes_client.get_album(album.id)
if not album_data: if not album_data:
continue continue
# get_album includes tracks by default (include_tracks=True)
tracks = album_data.get('tracks', {}).get('items', [])
tracks = self._extract_track_items(album_data)
logger.debug(f" Album {album_idx}: {album_data.get('name', 'Unknown')} ({len(tracks)} tracks)") logger.debug(f" Album {album_idx}: {album_data.get('name', 'Unknown')} ({len(tracks)} tracks)")
# Skip albums with placeholder tracks (unreleased tracklist)
if self._has_placeholder_tracks(tracks): if self._has_placeholder_tracks(tracks):
logger.info(f" Skipping album with placeholder tracks: {album_data.get('name', 'Unknown')}") logger.info(f" Skipping album with placeholder tracks: {album_data.get('name', 'Unknown')}")
continue continue
# Determine if this is a new release (within last 30 days)
is_new = False is_new = False
try: try:
release_date_str = album_data.get('release_date', '') release_date_str = album_data.get('release_date', '')
if release_date_str: if release_date_str and len(release_date_str) >= 10:
# Handle full date or year-only
if len(release_date_str) >= 10:
release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d") release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d")
days_old = (datetime.now() - release_date).days is_new = (datetime.now() - release_date).days <= 30
is_new = days_old <= 30 except Exception:
except:
pass pass
# Add each track to discovery pool
for track in tracks: for track in tracks:
try: try:
# Enhance track object with full album data (including album_type)
enhanced_track = { enhanced_track = {
**track, **track,
'album': { 'album': {
@ -2510,17 +2562,14 @@ class WatchlistScanner:
'album_type': album_data.get('album_type', 'album'), 'album_type': album_data.get('album_type', 'album'),
'total_tracks': album_data.get('total_tracks', 0) 'total_tracks': album_data.get('total_tracks', 0)
}, },
'_source': source '_source': selected_source
} }
# Build track data for discovery pool with source-specific IDs
# iTunes/Deezer have no popularity data — synthesize from recency + occurrence
raw_popularity = album_data.get('popularity', 0) raw_popularity = album_data.get('popularity', 0)
if source in ('itunes', 'deezer') and raw_popularity == 0: if selected_source in ('itunes', 'deezer') and raw_popularity == 0:
# Base 45, boost by recency and artist occurrence count
synth_pop = 45 synth_pop = 45
if is_new: if is_new:
synth_pop += 25 # New releases get a big boost synth_pop += 25
else: else:
try: try:
release_str = album_data.get('release_date', '') release_str = album_data.get('release_date', '')
@ -2531,9 +2580,8 @@ class WatchlistScanner:
synth_pop += 15 synth_pop += 15
elif age_days <= 365: elif age_days <= 365:
synth_pop += 5 synth_pop += 5
except: except Exception:
pass pass
# Artists that appear similar to multiple watchlist artists are likely more relevant
if similar_artist.occurrence_count >= 3: if similar_artist.occurrence_count >= 3:
synth_pop += 10 synth_pop += 10
elif similar_artist.occurrence_count >= 2: elif similar_artist.occurrence_count >= 2:
@ -2553,40 +2601,30 @@ class WatchlistScanner:
'artist_genres': artist_genres 'artist_genres': artist_genres
} }
# Add source-specific IDs if selected_source == 'spotify':
if source == 'spotify':
track_data['spotify_track_id'] = track.get('id') track_data['spotify_track_id'] = track.get('id')
track_data['spotify_album_id'] = album_data.get('id') track_data['spotify_album_id'] = album_data.get('id')
track_data['spotify_artist_id'] = similar_artist.similar_artist_spotify_id track_data['spotify_artist_id'] = selected_artist_id
elif source == 'deezer': elif selected_source == 'deezer':
track_data['deezer_track_id'] = track.get('id') track_data['deezer_track_id'] = track.get('id')
track_data['deezer_album_id'] = album_data.get('id') track_data['deezer_album_id'] = album_data.get('id')
track_data['deezer_artist_id'] = getattr(similar_artist, 'similar_artist_deezer_id', None) track_data['deezer_artist_id'] = selected_artist_id
else: # itunes else:
track_data['itunes_track_id'] = track.get('id') track_data['itunes_track_id'] = track.get('id')
track_data['itunes_album_id'] = album_data.get('id') track_data['itunes_album_id'] = album_data.get('id')
track_data['itunes_artist_id'] = similar_artist.similar_artist_itunes_id track_data['itunes_artist_id'] = selected_artist_id
# Add to discovery pool with source (scoped to profile) if self.database.add_to_discovery_pool(track_data, source=selected_source, profile_id=profile_id):
if self.database.add_to_discovery_pool(track_data, source=source, profile_id=profile_id):
total_tracks_added += 1 total_tracks_added += 1
except Exception as track_error: except Exception as track_error:
logger.debug(f"Error adding track to discovery pool: {track_error}") logger.debug(f"Error adding track to discovery pool: {track_error}")
continue continue
# Small delay between albums
time.sleep(DELAY_BETWEEN_ALBUMS) time.sleep(DELAY_BETWEEN_ALBUMS)
except Exception as album_error: except Exception as album_error:
logger.warning(f"Error processing album on {source}: {album_error}") logger.warning(f"Error processing album on {selected_source}: {album_error}")
continue continue
except Exception as source_error:
logger.warning(f"Error processing {source} source for {similar_artist.similar_artist_name}: {source_error}")
continue
# Delay between artists (after processing all sources for this artist)
if artist_idx < len(similar_artists): if artist_idx < len(similar_artists):
time.sleep(DELAY_BETWEEN_ARTISTS) time.sleep(DELAY_BETWEEN_ARTISTS)
@ -2625,67 +2663,48 @@ class WatchlistScanner:
db_source = None db_source = None
artist_id_for_genres = None artist_id_for_genres = None
# Try Spotify first if available for source in discovery_sources:
if spotify_available:
try: try:
search_results = self.spotify_client.search_albums( search_query = query if source != 'spotify' else f"album:{album_row['title']} artist:{album_row['artist_name']}"
f"album:{album_row['title']} artist:{album_row['artist_name']}", search_results = self._search_albums_for_source(source, search_query, limit=1)
limit=1, if not search_results:
allow_fallback=False, continue
)
if search_results and len(search_results) > 0:
spotify_album = search_results[0]
album_data = self.spotify_client.get_album(spotify_album.id)
if album_data and 'tracks' in album_data:
tracks = album_data['tracks'].get('items', [])
db_source = 'spotify'
if album_data.get('artists'):
artist_id_for_genres = album_data['artists'][0]['id']
except Exception as e:
logger.debug(f"Spotify search failed for {album_row['title']}: {e}")
# Fall back to fallback source (iTunes/Deezer) if Spotify didn't work album_candidate = search_results[0]
if not tracks and fallback_available: album_data = self._get_album_data_for_source(source, album_candidate.id, album_name=album_row['title'])
try: if not album_data:
search_results = itunes_client.search_albums(query, limit=1) continue
if search_results and len(search_results) > 0:
fallback_album = search_results[0] tracks = self._extract_track_items(album_data)
album_data = itunes_client.get_album(fallback_album.id) if not tracks:
if album_data: continue
tracks_data = itunes_client.get_album_tracks(fallback_album.id)
tracks = tracks_data.get('items', []) if tracks_data else [] db_source = source
db_source = fallback_source
# Artist ID is in the album data
if album_data.get('artists'): if album_data.get('artists'):
artist_id_for_genres = album_data['artists'][0].get('id') artist_id_for_genres = album_data['artists'][0].get('id')
break
except Exception as e: except Exception as e:
logger.debug(f"{fallback_source} search failed for {album_row['title']}: {e}") logger.debug(f"{source} search failed for {album_row['title']}: {e}")
if not tracks or not album_data: if not tracks or not album_data:
continue continue
# Fetch artist genres
artist_genres = [] artist_genres = []
try: try:
if artist_id_for_genres: if artist_id_for_genres:
if db_source == 'spotify': artist_data = self._get_artist_data_for_source(db_source, artist_id_for_genres)
artist_data = self.spotify_client.get_artist(artist_id_for_genres)
else:
artist_data = itunes_client.get_artist(artist_id_for_genres)
if artist_data and 'genres' in artist_data: if artist_data and 'genres' in artist_data:
artist_genres = artist_data['genres'] artist_genres = artist_data['genres']
except Exception as e: except Exception as e:
logger.debug(f"Could not fetch genres for album artist: {e}") logger.debug(f"Could not fetch genres for album artist: {e}")
# Check if new release
is_new = False is_new = False
try: try:
release_date_str = album_data.get('release_date', '') release_date_str = album_data.get('release_date', '')
if release_date_str and len(release_date_str) >= 10: if release_date_str and len(release_date_str) >= 10:
release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d") release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d")
days_old = (datetime.now() - release_date).days is_new = (datetime.now() - release_date).days <= 30
is_new = days_old <= 30 except Exception:
except:
pass pass
for track in tracks: for track in tracks:
@ -2716,7 +2735,6 @@ class WatchlistScanner:
'artist_genres': artist_genres 'artist_genres': artist_genres
} }
# Add source-specific IDs
if db_source == 'spotify': if db_source == 'spotify':
track_data['spotify_track_id'] = track.get('id') track_data['spotify_track_id'] = track.get('id')
track_data['spotify_album_id'] = album_data.get('id') track_data['spotify_album_id'] = album_data.get('id')
@ -2725,14 +2743,14 @@ class WatchlistScanner:
track_data['deezer_track_id'] = track.get('id') track_data['deezer_track_id'] = track.get('id')
track_data['deezer_album_id'] = album_data.get('id') track_data['deezer_album_id'] = album_data.get('id')
track_data['deezer_artist_id'] = artist_id_for_genres or '' track_data['deezer_artist_id'] = artist_id_for_genres or ''
else: # itunes else:
track_data['itunes_track_id'] = track.get('id') track_data['itunes_track_id'] = track.get('id')
track_data['itunes_album_id'] = album_data.get('id') track_data['itunes_album_id'] = album_data.get('id')
track_data['itunes_artist_id'] = artist_id_for_genres or '' track_data['itunes_artist_id'] = artist_id_for_genres or ''
if self.database.add_to_discovery_pool(track_data, source=db_source, profile_id=profile_id): if self.database.add_to_discovery_pool(track_data, source=db_source, profile_id=profile_id):
total_tracks_added += 1 total_tracks_added += 1
except Exception as track_error: except Exception:
continue continue
time.sleep(DELAY_BETWEEN_ALBUMS) time.sleep(DELAY_BETWEEN_ALBUMS)
@ -2819,6 +2837,7 @@ class WatchlistScanner:
album_type='album,single,ep', album_type='album,single,ep',
limit=5, limit=5,
skip_cache=True, skip_cache=True,
max_pages=1,
) )
if not recent_releases: if not recent_releases:
@ -3060,6 +3079,7 @@ class WatchlistScanner:
album_type='album,single,ep', album_type='album,single,ep',
limit=20, limit=20,
skip_cache=True, skip_cache=True,
max_pages=2,
) )
for album in albums or []: for album in albums or []:
process_album(album, artist.artist_name, artist.spotify_artist_id, fallback_id if fallback_source == 'itunes' else None, 'spotify') process_album(album, artist.artist_name, artist.spotify_artist_id, fallback_id if fallback_source == 'itunes' else None, 'spotify')
@ -3117,6 +3137,7 @@ class WatchlistScanner:
album_type='album,single,ep', album_type='album,single,ep',
limit=20, limit=20,
skip_cache=True, skip_cache=True,
max_pages=2,
) )
for album in albums or []: for album in albums or []:
process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, fallback_id if fallback_source == 'itunes' else None, 'spotify') process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, fallback_id if fallback_source == 'itunes' else None, 'spotify')

335
tests/test_watchlist_scanner_scan.py
View file

@ -76,11 +76,14 @@ class _FakeMetadataService:
class _FakeSourceClient: class _FakeSourceClient:
def __init__(self, *, artist_id: str, albums, image_url: str): def __init__(self, *, artist_id: str, albums, image_url: str, album_payload=None, album_search_results=None):
self.artist_id = artist_id self.artist_id = artist_id
self.albums = list(albums) self.albums = list(albums)
self.image_url = image_url self.image_url = image_url
self.album_payload = album_payload
self.album_search_results = list(album_search_results or [])
self.search_calls = [] self.search_calls = []
self.search_album_calls = []
self.album_calls = [] self.album_calls = []
self.artist_calls = [] self.artist_calls = []
@ -88,22 +91,41 @@ class _FakeSourceClient:
self.search_calls.append((query, limit, kwargs)) self.search_calls.append((query, limit, kwargs))
return [types.SimpleNamespace(id=self.artist_id, name=query)] return [types.SimpleNamespace(id=self.artist_id, name=query)]
def search_albums(self, query, limit=1, **kwargs):
self.search_album_calls.append((query, limit, kwargs))
return list(self.album_search_results)
def get_artist_albums(self, artist_id, album_type='album,single', limit=50, **kwargs): def get_artist_albums(self, artist_id, album_type='album,single', limit=50, **kwargs):
self.album_calls.append((artist_id, album_type, limit, kwargs)) self.album_calls.append((artist_id, album_type, limit, kwargs))
return list(self.albums) return list(self.albums)
def get_artist(self, artist_id): def get_artist(self, artist_id, **kwargs):
self.artist_calls.append(artist_id) self.artist_calls.append(artist_id)
return { return {
"id": artist_id, "id": artist_id,
"images": [{"url": self.image_url}] if self.image_url else [], "images": [{"url": self.image_url}] if self.image_url else [],
} }
def get_album(self, album_id, **kwargs):
self.album_calls.append((album_id, kwargs))
if self.album_payload is not None:
return self.album_payload
return {
"id": album_id,
"name": "Album One",
"images": [{"url": self.image_url}] if self.image_url else [],
"tracks": {"items": []},
"artists": [{"id": self.artist_id}],
}
class _FakeDB: class _FakeDB:
def __init__(self, artists): def __init__(self, artists):
self.artists = artists self.artists = artists
self.similar_calls = [] self.similar_calls = []
self.discovery_pool_calls = []
self.discovery_pool_timestamp_calls = []
self.db_albums = []
def get_watchlist_artists(self, profile_id=None): def get_watchlist_artists(self, profile_id=None):
return list(self.artists) return list(self.artists)
@ -112,6 +134,52 @@ class _FakeDB:
self.similar_calls.append((args, kwargs)) self.similar_calls.append((args, kwargs))
return False return False
def should_populate_discovery_pool(self, hours_threshold=24, profile_id=1):
return True
def get_top_similar_artists(self, limit=50, profile_id=1):
return []
def add_to_discovery_pool(self, track_data, source, profile_id=1):
self.discovery_pool_calls.append((track_data, source, profile_id))
return True
def cleanup_old_discovery_tracks(self, days_threshold=365):
return 0
def update_discovery_pool_timestamp(self, track_count, profile_id=1):
self.discovery_pool_timestamp_calls.append((track_count, profile_id))
return True
class _Cursor:
def __init__(self, parent):
self.parent = parent
def execute(self, *args, **kwargs):
return None
def fetchall(self):
return list(self.parent.db_albums)
def fetchone(self):
return {"count": 0}
class _Conn:
def __init__(self, cursor):
self._cursor = cursor
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def cursor(self):
return self._cursor
def _get_connection(self):
return self._Conn(self._Cursor(self))
def _build_artist(name="Artist One", profile_id=11): def _build_artist(name="Artist One", profile_id=11):
return types.SimpleNamespace( return types.SimpleNamespace(
@ -384,6 +452,269 @@ def test_get_artist_discography_for_watchlist_falls_back_when_primary_empty(monk
assert spotify_client.album_calls assert spotify_client.album_calls
def test_populate_discovery_pool_uses_primary_source_first(monkeypatch):
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0)
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0)
monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None))
monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
similar_artist = types.SimpleNamespace(
id=501,
similar_artist_name="Similar Artist",
occurrence_count=3,
similar_artist_spotify_id="sp-artist",
similar_artist_itunes_id="it-artist",
similar_artist_deezer_id="dz-artist",
)
album = types.SimpleNamespace(id="dz-album-1", name="Deezer Album", album_type="album")
deezer_album_payload = {
"id": "dz-album-1",
"name": "Deezer Album",
"images": [{"url": "https://example.com/deezer-album.jpg"}],
"release_date": "2026-04-01",
"popularity": 0,
"tracks": {
"items": [
{
"id": "dz-track-1",
"name": "Deezer Track",
"duration_ms": 123456,
"artists": [{"name": "Similar Artist"}],
}
]
},
"artists": [{"id": "dz-artist"}],
}
deezer_client = _FakeSourceClient(
artist_id="dz-artist",
albums=[album],
image_url="https://example.com/deezer-artist.jpg",
album_payload=deezer_album_payload,
)
spotify_client = _FakeSourceClient(
artist_id="sp-artist",
albums=[types.SimpleNamespace(id="sp-album-1", name="Spotify Album", album_type="album")],
image_url="https://example.com/spotify-artist.jpg",
album_payload={
"id": "sp-album-1",
"name": "Spotify Album",
"images": [{"url": "https://example.com/spotify-album.jpg"}],
"release_date": "2026-04-01",
"popularity": 50,
"tracks": {"items": []},
"artists": [{"id": "sp-artist"}],
},
)
def fake_get_client_for_source(source):
return {
"deezer": deezer_client,
"spotify": spotify_client,
}.get(source)
monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source)
scanner = _build_scanner({"tracks": {"items": []}}, [])
scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False
scanner.database.should_populate_discovery_pool = lambda hours_threshold=24, profile_id=1: True
scanner.database.get_top_similar_artists = lambda limit=50, profile_id=1: [similar_artist]
scanner.database.db_albums = []
scanner.cache_discovery_recent_albums = lambda *args, **kwargs: None
scanner.curate_discovery_playlists = lambda *args, **kwargs: None
scanner.database.update_discovery_pool_timestamp = lambda *args, **kwargs: True
scanner.database.cleanup_old_discovery_tracks = lambda *args, **kwargs: 0
scanner.populate_discovery_pool(top_artists_limit=1, albums_per_artist=1, profile_id=1)
assert scanner.database.discovery_pool_calls
assert scanner.database.discovery_pool_calls[0][1] == "deezer"
assert deezer_client.album_calls
assert spotify_client.search_calls == []
assert spotify_client.artist_calls == []
def test_populate_discovery_pool_falls_back_to_spotify_when_primary_has_no_albums(monkeypatch):
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0)
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0)
monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None))
monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
similar_artist = types.SimpleNamespace(
id=502,
similar_artist_name="Fallback Artist",
occurrence_count=1,
similar_artist_spotify_id="sp-artist",
similar_artist_itunes_id="it-artist",
similar_artist_deezer_id="dz-artist",
)
deezer_client = _FakeSourceClient(
artist_id="dz-artist",
albums=[],
image_url="https://example.com/deezer-artist.jpg",
)
spotify_album = types.SimpleNamespace(id="sp-album-1", name="Spotify Album", album_type="album")
spotify_client = _FakeSourceClient(
artist_id="sp-artist",
albums=[spotify_album],
image_url="https://example.com/spotify-artist.jpg",
album_payload={
"id": "sp-album-1",
"name": "Spotify Album",
"images": [{"url": "https://example.com/spotify-album.jpg"}],
"release_date": "2026-04-01",
"popularity": 50,
"tracks": {
"items": [
{
"id": "sp-track-1",
"name": "Spotify Track",
"duration_ms": 234567,
"artists": [{"name": "Fallback Artist"}],
}
]
},
"artists": [{"id": "sp-artist"}],
},
)
def fake_get_client_for_source(source):
return {
"deezer": deezer_client,
"spotify": spotify_client,
}.get(source)
monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source)
scanner = _build_scanner({"tracks": {"items": []}}, [])
scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False
scanner.database.should_populate_discovery_pool = lambda hours_threshold=24, profile_id=1: True
scanner.database.get_top_similar_artists = lambda limit=50, profile_id=1: [similar_artist]
scanner.database.db_albums = []
scanner.cache_discovery_recent_albums = lambda *args, **kwargs: None
scanner.curate_discovery_playlists = lambda *args, **kwargs: None
scanner.database.update_discovery_pool_timestamp = lambda *args, **kwargs: True
scanner.database.cleanup_old_discovery_tracks = lambda *args, **kwargs: 0
scanner.populate_discovery_pool(top_artists_limit=1, albums_per_artist=1, profile_id=1)
assert scanner.database.discovery_pool_calls
assert scanner.database.discovery_pool_calls[0][1] == "spotify"
assert deezer_client.album_calls
assert spotify_client.search_calls == [("Fallback Artist", 1, {"allow_fallback": False})]
assert spotify_client.album_calls
assert any(
isinstance(call, tuple)
and len(call) == 4
and call[0] == "sp-artist"
and call[3].get("skip_cache") is False
and call[3].get("allow_fallback") is False
and call[3].get("max_pages") == 2
for call in spotify_client.album_calls
)
assert any(
isinstance(call, tuple)
and len(call) == 4
and call[3].get("allow_fallback") is False
for call in spotify_client.album_calls
)
def test_populate_discovery_pool_uses_strict_spotify_for_database_album_search(monkeypatch):
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0)
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0)
monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None))
monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
similar_artist = types.SimpleNamespace(
id=503,
similar_artist_name="No Album Artist",
occurrence_count=1,
similar_artist_spotify_id="sp-artist",
similar_artist_itunes_id="it-artist",
similar_artist_deezer_id="dz-artist",
)
deezer_client = _FakeSourceClient(
artist_id="dz-artist",
albums=[],
image_url="https://example.com/deezer-artist.jpg",
)
spotify_client = _FakeSourceClient(
artist_id="sp-artist",
albums=[],
image_url="https://example.com/spotify-artist.jpg",
album_search_results=[types.SimpleNamespace(id="sp-db-album", name="DB Album")],
album_payload={
"id": "sp-db-album",
"name": "DB Album",
"images": [{"url": "https://example.com/db-album.jpg"}],
"release_date": "2026-04-01",
"popularity": 75,
"tracks": {
"items": [
{
"id": "sp-db-track-1",
"name": "DB Track",
"duration_ms": 345678,
"artists": [{"name": "DB Artist"}],
}
]
},
"artists": [{"id": "sp-artist"}],
},
)
def fake_get_client_for_source(source):
return {
"deezer": deezer_client,
"spotify": spotify_client,
}.get(source)
monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source)
scanner = _build_scanner({"tracks": {"items": []}}, [])
scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False
scanner.database.should_populate_discovery_pool = lambda hours_threshold=24, profile_id=1: True
scanner.database.get_top_similar_artists = lambda limit=50, profile_id=1: [similar_artist]
scanner.database.db_albums = [{"title": "DB Album", "artist_name": "DB Artist"}]
scanner.cache_discovery_recent_albums = lambda *args, **kwargs: None
scanner.curate_discovery_playlists = lambda *args, **kwargs: None
scanner.database.update_discovery_pool_timestamp = lambda *args, **kwargs: True
scanner.database.cleanup_old_discovery_tracks = lambda *args, **kwargs: 0
scanner.populate_discovery_pool(top_artists_limit=1, albums_per_artist=1, profile_id=1)
assert scanner.database.discovery_pool_calls
assert scanner.database.discovery_pool_calls[0][1] == "spotify"
assert spotify_client.search_album_calls
assert any(
kwargs.get("allow_fallback") is False
for _, _, kwargs in spotify_client.search_album_calls
)
assert any(
isinstance(call, tuple)
and len(call) == 4
and call[0] == "sp-artist"
and call[3].get("skip_cache") is False
and call[3].get("allow_fallback") is False
and call[3].get("max_pages") == 2
for call in spotify_client.album_calls
)
assert any(
isinstance(call, tuple)
and len(call) == 2
and call[1].get("allow_fallback") is False
for call in spotify_client.album_calls
if len(call) == 2
)
def test_match_to_spotify_uses_strict_lookup(): def test_match_to_spotify_uses_strict_lookup():
spotify_client = _FakeSpotifyClient( spotify_client = _FakeSpotifyClient(
search_results=[types.SimpleNamespace(id="fallback-id", name="Artist One")] search_results=[types.SimpleNamespace(id="fallback-id", name="Artist One")]