danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Reduce discovery fan-out and pagination

Browse source 
Make discovery pool population respect provider priority while keeping Spotify strict, and reduce unnecessary request volume in the hot discovery paths.

- keep discovery fan-out source-priority aware
- preserve cache use where freshness is not required
- cap Spotify artist-album pagination in discovery and cache refresh paths
- keep incremental release checks to a single page, since they only need the newest releases
- add regression coverage for provider order, strict Spotify handling, and pagination caps
This commit is contained in:
Antti Kettunen 2026-04-16 20:59:26 +03:00
parent 0b60986f44
commit e447cf6ab0
3 changed files with 637 additions and 283 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
core/metadata_service.py
View file

@ -128,6 +128,8 @@ def get_album_tracks_for_source(source: str, album_id: str):
fetch = getattr(client, 'get_album_tracks_dict', None) if source == 'hydrabase' else getattr(client, 'get_album_tracks', None) fetch = getattr(client, 'get_album_tracks_dict', None) if source == 'hydrabase' else getattr(client, 'get_album_tracks', None)
if not fetch: if not fetch:
return None return None
if source == 'spotify':
return fetch(album_id, allow_fallback=False)
return fetch(album_id) return fetch(album_id)
except Exception: except Exception:
return None return None

583
core/watchlist_scanner.py
View file

@ -451,6 +451,14 @@ class WatchlistScanner:
"""Return watchlist scan sources in the configured priority order.""" """Return watchlist scan sources in the configured priority order."""
return list(get_source_priority(get_primary_source())) return list(get_source_priority(get_primary_source()))
def _discovery_source_priority(self) -> List[str]:
"""Return discovery sources in configured priority order.
Discovery pool writes only support Spotify, iTunes, and Deezer IDs, so
we filter the broader metadata priority list down to those sources.
"""
return [source for source in self._watchlist_source_priority() if source in {'spotify', 'itunes', 'deezer'}]
@staticmethod @staticmethod
def _artist_id_attribute_for_source(source: str) -> Optional[str]: def _artist_id_attribute_for_source(source: str) -> Optional[str]:
"""Return the watchlist artist attribute that stores the given source ID.""" """Return the watchlist artist attribute that stores the given source ID."""
@ -501,7 +509,10 @@ class WatchlistScanner:
return None return None
try: try:
search_results = client.search_artists(watchlist_artist.artist_name, limit=1) search_kwargs = {'limit': 1}
if source == 'spotify':
search_kwargs['allow_fallback'] = False
search_results = client.search_artists(watchlist_artist.artist_name, **search_kwargs)
except Exception as e: except Exception as e:
logger.debug("Could not search %s for %s: %s", source, watchlist_artist.artist_name, e) logger.debug("Could not search %s for %s: %s", source, watchlist_artist.artist_name, e)
return None return None
@ -552,7 +563,10 @@ class WatchlistScanner:
return None return None
try: try:
artist_data = client.get_artist(artist_id) if source == 'spotify':
artist_data = client.get_artist(artist_id, allow_fallback=False)
else:
artist_data = client.get_artist(artist_id)
except Exception as e: except Exception as e:
logger.debug("Could not fetch artist image for %s on %s: %s", watchlist_artist.artist_name, source, e) logger.debug("Could not fetch artist image for %s on %s: %s", watchlist_artist.artist_name, source, e)
return None return None
@ -566,7 +580,10 @@ class WatchlistScanner:
return None return None
try: try:
album_data = client.get_album(album_id) if source == 'spotify':
album_data = client.get_album(album_id, allow_fallback=False)
else:
album_data = client.get_album(album_id)
except Exception as e: except Exception as e:
logger.debug("Could not fetch album %s on %s: %s", album_id, source, e) logger.debug("Could not fetch album %s on %s: %s", album_id, source, e)
album_data = None album_data = None
@ -666,6 +683,97 @@ class WatchlistScanner:
return image_url return image_url
return None return None
def _get_artist_albums_for_source(
self,
source: str,
artist_id: str,
album_type: str = 'album,single,ep',
limit: int = 50,
skip_cache: bool = True,
max_pages: int = 0,
) -> List[Any]:
"""Fetch artist albums for a specific source, keeping Spotify strict."""
client = get_client_for_source(source)
if not client or not artist_id or not hasattr(client, 'get_artist_albums'):
return []
try:
kwargs = {
'album_type': album_type,
'limit': limit,
}
if source == 'spotify':
kwargs['skip_cache'] = skip_cache
kwargs['max_pages'] = max_pages
kwargs['allow_fallback'] = False
return client.get_artist_albums(artist_id, **kwargs) or []
except Exception as e:
logger.debug("Could not fetch artist albums for %s on %s: %s", artist_id, source, e)
return []
def _get_artist_data_for_source(self, source: str, artist_id: str) -> Optional[Dict[str, Any]]:
"""Fetch artist metadata for a specific source, keeping Spotify strict."""
client = get_client_for_source(source)
if not client or not artist_id or not hasattr(client, 'get_artist'):
return None
try:
if source == 'spotify':
return client.get_artist(artist_id, allow_fallback=False)
return client.get_artist(artist_id)
except Exception as e:
logger.debug("Could not fetch artist data for %s on %s: %s", artist_id, source, e)
return None
def _search_albums_for_source(self, source: str, query: str, limit: int = 1):
"""Search albums for a specific source, keeping Spotify strict."""
client = get_client_for_source(source)
if not client or not hasattr(client, 'search_albums'):
return []
try:
if source == 'spotify':
return client.search_albums(query, limit=limit, allow_fallback=False) or []
return client.search_albums(query, limit=limit) or []
except Exception as e:
logger.debug("Could not search albums for %s on %s: %s", query, source, e)
return []
def _resolve_artist_id_for_source(
self,
source: str,
artist_name: str,
stored_id: Optional[str] = None,
cache_callback: Optional[Callable[[str], None]] = None,
) -> Optional[str]:
"""Resolve an artist ID for a specific source, searching by name if needed."""
if stored_id:
return stored_id
client = get_client_for_source(source)
if not client or not hasattr(client, 'search_artists'):
return None
try:
search_kwargs = {'limit': 1}
if source == 'spotify':
search_kwargs['allow_fallback'] = False
results = client.search_artists(artist_name, **search_kwargs)
except Exception as e:
logger.debug("Could not resolve %s artist ID for %s: %s", source, artist_name, e)
return None
if not results:
return None
found_id = self._extract_entity_id(results[0])
if found_id and cache_callback:
try:
cache_callback(found_id)
except Exception as e:
logger.debug("Could not cache %s artist ID for %s: %s", source, artist_name, e)
return found_id
def backfill_watchlist_artist_images(self, profile_id: int) -> int: def backfill_watchlist_artist_images(self, profile_id: int) -> int:
"""Backfill missing watchlist artist images using cached metadata and existing album art.""" """Backfill missing watchlist artist images using cached metadata and existing album art."""
try: try:
@ -2298,9 +2406,6 @@ class WatchlistScanner:
from datetime import datetime, timedelta from datetime import datetime, timedelta
import random import random
if self.spotify_client and self.spotify_client.is_rate_limited():
self._disable_spotify_for_run("global Spotify rate limit active")
# Check if we should run discovery pool population (prevents over-polling) # Check if we should run discovery pool population (prevents over-polling)
skip_pool_population = not self.database.should_populate_discovery_pool(hours_threshold=24, profile_id=profile_id) skip_pool_population = not self.database.should_populate_discovery_pool(hours_threshold=24, profile_id=profile_id)
@ -2320,18 +2425,12 @@ class WatchlistScanner:
logger.info("Populating discovery pool from similar artists...") logger.info("Populating discovery pool from similar artists...")
# Determine which sources are available discovery_sources = self._discovery_source_priority()
spotify_available = self._spotify_is_primary_source() if not discovery_sources:
# Import fallback metadata client (iTunes or Deezer)
itunes_client, fallback_source = _get_fallback_metadata_client()
fallback_available = True # Fallback source is always available (no auth needed)
if not spotify_available and not fallback_available:
logger.warning("No music sources available to populate discovery pool") logger.warning("No music sources available to populate discovery pool")
return return
logger.info(f"Sources available - Spotify: {spotify_available}, {fallback_source}: {fallback_available}") logger.info("Discovery source priority: %s", discovery_sources)
# Get top similar artists for this profile's watchlist (ordered by occurrence_count) # Get top similar artists for this profile's watchlist (ordered by occurrence_count)
similar_artists = self.database.get_top_similar_artists(limit=top_artists_limit, profile_id=profile_id) similar_artists = self.database.get_top_similar_artists(limit=top_artists_limit, profile_id=profile_id)
@ -2360,233 +2459,172 @@ class WatchlistScanner:
if progress_callback: if progress_callback:
progress_callback('artist', f'{similar_artist.similar_artist_name} ({artist_idx}/{len(similar_artists)})') progress_callback('artist', f'{similar_artist.similar_artist_name} ({artist_idx}/{len(similar_artists)})')
# Build list of sources to process for this artist # Resolve the first source that can actually produce albums.
# Fallback source (iTunes/Deezer) is ALWAYS processed (baseline), Spotify is added if authenticated selected_source = None
sources_to_process = [] selected_artist_id = None
selected_albums = []
artist_genres: List[str] = []
# Always add fallback source first (baseline source) for source in discovery_sources:
fallback_id = similar_artist.similar_artist_itunes_id if fallback_source == 'itunes' else getattr(similar_artist, 'similar_artist_deezer_id', None) source_attr = self._artist_id_attribute_for_source(source)
if not fallback_id: stored_id = getattr(similar_artist, source_attr, None) if source_attr else None
# On-the-fly lookup for missing fallback ID (seamless provider switching)
try:
fallback_results = itunes_client.search_artists(similar_artist.similar_artist_name, limit=1)
if fallback_results and len(fallback_results) > 0:
fallback_id = fallback_results[0].id
# Cache it for future use
if fallback_source == 'deezer':
self.database.update_similar_artist_deezer_id(similar_artist.id, fallback_id)
else:
self.database.update_similar_artist_itunes_id(similar_artist.id, fallback_id)
logger.debug(f" Resolved {fallback_source} ID {fallback_id} for {similar_artist.similar_artist_name}")
except Exception as e:
logger.debug(f" Could not resolve {fallback_source} ID for {similar_artist.similar_artist_name}: {e}")
if fallback_id: cache_callback = None
sources_to_process.append((fallback_source, fallback_id)) if source == 'itunes':
cache_callback = lambda found_id, artist_id=similar_artist.id: self.database.update_similar_artist_itunes_id(artist_id, found_id)
elif source == 'deezer':
cache_callback = lambda found_id, artist_id=similar_artist.id: self.database.update_similar_artist_deezer_id(artist_id, found_id)
# Add Spotify if authenticated and we have an ID artist_id = self._resolve_artist_id_for_source(
if spotify_available and similar_artist.similar_artist_spotify_id: source,
sources_to_process.append(('spotify', similar_artist.similar_artist_spotify_id)) similar_artist.similar_artist_name,
stored_id=stored_id,
if not sources_to_process: cache_callback=cache_callback,
logger.debug(f"No valid IDs for {similar_artist.similar_artist_name}, skipping") )
continue if not artist_id:
continue
logger.debug(f" Processing {len(sources_to_process)} source(s): {[s[0] for s in sources_to_process]}")
all_albums = self._get_artist_albums_for_source(
# Process each source for this artist source,
for source, artist_id in sources_to_process: artist_id,
try: album_type='album,single,ep',
# Get artist's albums from this source limit=50,
if source == 'spotify': skip_cache=False,
all_albums = self.spotify_client.get_artist_albums( max_pages=2,
artist_id, )
album_type='album,single,ep', if not all_albums:
limit=50, logger.debug(f"No albums found for {similar_artist.similar_artist_name} on {source}")
skip_cache=True, continue
)
else: # itunes or deezer fallback artist_data = self._get_artist_data_for_source(source, artist_id)
all_albums = itunes_client.get_artist_albums( if artist_data and 'genres' in artist_data:
artist_id, artist_genres = artist_data['genres']
album_type='album,single,ep',
limit=50 albums = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type == 'album']
) singles_eps = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type in ['single', 'ep']]
selected_albums = []
if not all_albums:
logger.debug(f"No albums found for {similar_artist.similar_artist_name} on {source}") latest_releases = all_albums[:3]
continue selected_albums.extend(latest_releases)
# Fetch artist genres for this source remaining_slots = albums_per_artist - len(selected_albums)
artist_genres = [] if remaining_slots > 0:
try: remaining_content = all_albums[3:]
if source == 'spotify': if len(remaining_content) > remaining_slots:
artist_data = self.spotify_client.get_artist(artist_id) selected_albums.extend(random.sample(remaining_content, remaining_slots))
if artist_data and 'genres' in artist_data: else:
artist_genres = artist_data['genres'] selected_albums.extend(remaining_content)
else: # itunes/deezer - genres from artist lookup
artist_data = itunes_client.get_artist(artist_id) selected_source = source
if artist_data and 'genres' in artist_data: selected_artist_id = artist_id
artist_genres = artist_data['genres'] logger.info(
except Exception as e: f" [{source}] Selected {len(selected_albums)} releases from {len(all_albums)} available "
logger.debug(f"Could not fetch genres for {similar_artist.similar_artist_name} on {source}: {e}") f"(albums: {len(albums)}, singles/EPs: {len(singles_eps)})"
)
# IMPROVED: Smart selection mixing albums, singles, and EPs break
# Prioritize recent releases and popular content
if not selected_source or not selected_artist_id or not selected_albums:
# Separate by type for balanced selection logger.debug(f"No valid source/albums for {similar_artist.similar_artist_name}, skipping")
albums = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type == 'album'] continue
singles_eps = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type in ['single', 'ep']]
other = [a for a in all_albums if not hasattr(a, 'album_type')] # Process each selected album from the winning source.
for album_idx, album in enumerate(selected_albums, 1):
# Select albums: latest releases + popular older content try:
selected_albums = [] album_data = self._get_album_data_for_source(selected_source, album.id, album_name=album.name)
if not album_data:
# Always include 3 most recent releases (any type) - this captures new singles/EPs continue
latest_releases = all_albums[:3]
selected_albums.extend(latest_releases) tracks = self._extract_track_items(album_data)
logger.debug(f" Album {album_idx}: {album_data.get('name', 'Unknown')} ({len(tracks)} tracks)")
# Add remaining slots with balanced mix
remaining_slots = albums_per_artist - len(selected_albums) if self._has_placeholder_tracks(tracks):
if remaining_slots > 0: logger.info(f" Skipping album with placeholder tracks: {album_data.get('name', 'Unknown')}")
# Combine remaining albums and singles continue
remaining_content = all_albums[3:]
is_new = False
if len(remaining_content) > remaining_slots: try:
# Randomly select from remaining content release_date_str = album_data.get('release_date', '')
random_selection = random.sample(remaining_content, remaining_slots) if release_date_str and len(release_date_str) >= 10:
selected_albums.extend(random_selection) release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d")
else: is_new = (datetime.now() - release_date).days <= 30
selected_albums.extend(remaining_content) except Exception:
pass
logger.info(f" [{source}] Selected {len(selected_albums)} releases from {len(all_albums)} available (albums: {len(albums)}, singles/EPs: {len(singles_eps)})")
for track in tracks:
# Process each selected album try:
for album_idx, album in enumerate(selected_albums, 1): enhanced_track = {
try: **track,
# Get full album data with tracks from appropriate source 'album': {
if source == 'spotify': 'id': album_data['id'],
album_data = self.spotify_client.get_album(album.id) 'name': album_data.get('name', 'Unknown Album'),
if not album_data or 'tracks' not in album_data: 'images': album_data.get('images', []),
continue 'release_date': album_data.get('release_date', ''),
tracks = album_data['tracks'].get('items', []) 'album_type': album_data.get('album_type', 'album'),
else: # itunes or deezer fallback 'total_tracks': album_data.get('total_tracks', 0)
album_data = itunes_client.get_album(album.id) },
if not album_data: '_source': selected_source
continue }
# get_album includes tracks by default (include_tracks=True)
tracks = album_data.get('tracks', {}).get('items', []) raw_popularity = album_data.get('popularity', 0)
if selected_source in ('itunes', 'deezer') and raw_popularity == 0:
logger.debug(f" Album {album_idx}: {album_data.get('name', 'Unknown')} ({len(tracks)} tracks)") synth_pop = 45
if is_new:
# Skip albums with placeholder tracks (unreleased tracklist) synth_pop += 25
if self._has_placeholder_tracks(tracks): else:
logger.info(f" Skipping album with placeholder tracks: {album_data.get('name', 'Unknown')}") try:
continue release_str = album_data.get('release_date', '')
if release_str and len(release_str) >= 10:
# Determine if this is a new release (within last 30 days) rel_date = datetime.strptime(release_str[:10], "%Y-%m-%d")
is_new = False age_days = (datetime.now() - rel_date).days
try: if age_days <= 90:
release_date_str = album_data.get('release_date', '') synth_pop += 15
if release_date_str: elif age_days <= 365:
# Handle full date or year-only synth_pop += 5
if len(release_date_str) >= 10: except Exception:
release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d") pass
days_old = (datetime.now() - release_date).days if similar_artist.occurrence_count >= 3:
is_new = days_old <= 30 synth_pop += 10
except: elif similar_artist.occurrence_count >= 2:
pass synth_pop += 5
raw_popularity = min(synth_pop, 100)
# Add each track to discovery pool
for track in tracks: track_data = {
try: 'track_name': track.get('name', 'Unknown Track'),
# Enhance track object with full album data (including album_type) 'artist_name': similar_artist.similar_artist_name,
enhanced_track = { 'album_name': album_data.get('name', 'Unknown Album'),
**track, 'album_cover_url': album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None,
'album': { 'duration_ms': track.get('duration_ms', 0),
'id': album_data['id'], 'popularity': raw_popularity,
'name': album_data.get('name', 'Unknown Album'), 'release_date': album_data.get('release_date', ''),
'images': album_data.get('images', []), 'is_new_release': is_new,
'release_date': album_data.get('release_date', ''), 'track_data_json': enhanced_track,
'album_type': album_data.get('album_type', 'album'), 'artist_genres': artist_genres
'total_tracks': album_data.get('total_tracks', 0) }
},
'_source': source if selected_source == 'spotify':
} track_data['spotify_track_id'] = track.get('id')
track_data['spotify_album_id'] = album_data.get('id')
# Build track data for discovery pool with source-specific IDs track_data['spotify_artist_id'] = selected_artist_id
# iTunes/Deezer have no popularity data — synthesize from recency + occurrence elif selected_source == 'deezer':
raw_popularity = album_data.get('popularity', 0) track_data['deezer_track_id'] = track.get('id')
if source in ('itunes', 'deezer') and raw_popularity == 0: track_data['deezer_album_id'] = album_data.get('id')
# Base 45, boost by recency and artist occurrence count track_data['deezer_artist_id'] = selected_artist_id
synth_pop = 45 else:
if is_new: track_data['itunes_track_id'] = track.get('id')
synth_pop += 25 # New releases get a big boost track_data['itunes_album_id'] = album_data.get('id')
else: track_data['itunes_artist_id'] = selected_artist_id
try:
release_str = album_data.get('release_date', '') if self.database.add_to_discovery_pool(track_data, source=selected_source, profile_id=profile_id):
if release_str and len(release_str) >= 10: total_tracks_added += 1
rel_date = datetime.strptime(release_str[:10], "%Y-%m-%d") except Exception as track_error:
age_days = (datetime.now() - rel_date).days logger.debug(f"Error adding track to discovery pool: {track_error}")
if age_days <= 90: continue
synth_pop += 15
elif age_days <= 365: time.sleep(DELAY_BETWEEN_ALBUMS)
synth_pop += 5 except Exception as album_error:
except: logger.warning(f"Error processing album on {selected_source}: {album_error}")
pass
# Artists that appear similar to multiple watchlist artists are likely more relevant
if similar_artist.occurrence_count >= 3:
synth_pop += 10
elif similar_artist.occurrence_count >= 2:
synth_pop += 5
raw_popularity = min(synth_pop, 100)
track_data = {
'track_name': track.get('name', 'Unknown Track'),
'artist_name': similar_artist.similar_artist_name,
'album_name': album_data.get('name', 'Unknown Album'),
'album_cover_url': album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None,
'duration_ms': track.get('duration_ms', 0),
'popularity': raw_popularity,
'release_date': album_data.get('release_date', ''),
'is_new_release': is_new,
'track_data_json': enhanced_track,
'artist_genres': artist_genres
}
# Add source-specific IDs
if source == 'spotify':
track_data['spotify_track_id'] = track.get('id')
track_data['spotify_album_id'] = album_data.get('id')
track_data['spotify_artist_id'] = similar_artist.similar_artist_spotify_id
elif source == 'deezer':
track_data['deezer_track_id'] = track.get('id')
track_data['deezer_album_id'] = album_data.get('id')
track_data['deezer_artist_id'] = getattr(similar_artist, 'similar_artist_deezer_id', None)
else: # itunes
track_data['itunes_track_id'] = track.get('id')
track_data['itunes_album_id'] = album_data.get('id')
track_data['itunes_artist_id'] = similar_artist.similar_artist_itunes_id
# Add to discovery pool with source (scoped to profile)
if self.database.add_to_discovery_pool(track_data, source=source, profile_id=profile_id):
total_tracks_added += 1
except Exception as track_error:
logger.debug(f"Error adding track to discovery pool: {track_error}")
continue
# Small delay between albums
time.sleep(DELAY_BETWEEN_ALBUMS)
except Exception as album_error:
logger.warning(f"Error processing album on {source}: {album_error}")
continue
except Exception as source_error:
logger.warning(f"Error processing {source} source for {similar_artist.similar_artist_name}: {source_error}")
continue continue
# Delay between artists (after processing all sources for this artist)
if artist_idx < len(similar_artists): if artist_idx < len(similar_artists):
time.sleep(DELAY_BETWEEN_ARTISTS) time.sleep(DELAY_BETWEEN_ARTISTS)
@ -2625,67 +2663,48 @@ class WatchlistScanner:
db_source = None db_source = None
artist_id_for_genres = None artist_id_for_genres = None
# Try Spotify first if available for source in discovery_sources:
if spotify_available:
try: try:
search_results = self.spotify_client.search_albums( search_query = query if source != 'spotify' else f"album:{album_row['title']} artist:{album_row['artist_name']}"
f"album:{album_row['title']} artist:{album_row['artist_name']}", search_results = self._search_albums_for_source(source, search_query, limit=1)
limit=1, if not search_results:
allow_fallback=False, continue
)
if search_results and len(search_results) > 0:
spotify_album = search_results[0]
album_data = self.spotify_client.get_album(spotify_album.id)
if album_data and 'tracks' in album_data:
tracks = album_data['tracks'].get('items', [])
db_source = 'spotify'
if album_data.get('artists'):
artist_id_for_genres = album_data['artists'][0]['id']
except Exception as e:
logger.debug(f"Spotify search failed for {album_row['title']}: {e}")
# Fall back to fallback source (iTunes/Deezer) if Spotify didn't work album_candidate = search_results[0]
if not tracks and fallback_available: album_data = self._get_album_data_for_source(source, album_candidate.id, album_name=album_row['title'])
try: if not album_data:
search_results = itunes_client.search_albums(query, limit=1) continue
if search_results and len(search_results) > 0:
fallback_album = search_results[0] tracks = self._extract_track_items(album_data)
album_data = itunes_client.get_album(fallback_album.id) if not tracks:
if album_data: continue
tracks_data = itunes_client.get_album_tracks(fallback_album.id)
tracks = tracks_data.get('items', []) if tracks_data else [] db_source = source
db_source = fallback_source if album_data.get('artists'):
# Artist ID is in the album data artist_id_for_genres = album_data['artists'][0].get('id')
if album_data.get('artists'): break
artist_id_for_genres = album_data['artists'][0].get('id')
except Exception as e: except Exception as e:
logger.debug(f"{fallback_source} search failed for {album_row['title']}: {e}") logger.debug(f"{source} search failed for {album_row['title']}: {e}")
if not tracks or not album_data: if not tracks or not album_data:
continue continue
# Fetch artist genres
artist_genres = [] artist_genres = []
try: try:
if artist_id_for_genres: if artist_id_for_genres:
if db_source == 'spotify': artist_data = self._get_artist_data_for_source(db_source, artist_id_for_genres)
artist_data = self.spotify_client.get_artist(artist_id_for_genres)
else:
artist_data = itunes_client.get_artist(artist_id_for_genres)
if artist_data and 'genres' in artist_data: if artist_data and 'genres' in artist_data:
artist_genres = artist_data['genres'] artist_genres = artist_data['genres']
except Exception as e: except Exception as e:
logger.debug(f"Could not fetch genres for album artist: {e}") logger.debug(f"Could not fetch genres for album artist: {e}")
# Check if new release
is_new = False is_new = False
try: try:
release_date_str = album_data.get('release_date', '') release_date_str = album_data.get('release_date', '')
if release_date_str and len(release_date_str) >= 10: if release_date_str and len(release_date_str) >= 10:
release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d") release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d")
days_old = (datetime.now() - release_date).days is_new = (datetime.now() - release_date).days <= 30
is_new = days_old <= 30 except Exception:
except:
pass pass
for track in tracks: for track in tracks:
@ -2716,7 +2735,6 @@ class WatchlistScanner:
'artist_genres': artist_genres 'artist_genres': artist_genres
} }
# Add source-specific IDs
if db_source == 'spotify': if db_source == 'spotify':
track_data['spotify_track_id'] = track.get('id') track_data['spotify_track_id'] = track.get('id')
track_data['spotify_album_id'] = album_data.get('id') track_data['spotify_album_id'] = album_data.get('id')
@ -2725,14 +2743,14 @@ class WatchlistScanner:
track_data['deezer_track_id'] = track.get('id') track_data['deezer_track_id'] = track.get('id')
track_data['deezer_album_id'] = album_data.get('id') track_data['deezer_album_id'] = album_data.get('id')
track_data['deezer_artist_id'] = artist_id_for_genres or '' track_data['deezer_artist_id'] = artist_id_for_genres or ''
else: # itunes else:
track_data['itunes_track_id'] = track.get('id') track_data['itunes_track_id'] = track.get('id')
track_data['itunes_album_id'] = album_data.get('id') track_data['itunes_album_id'] = album_data.get('id')
track_data['itunes_artist_id'] = artist_id_for_genres or '' track_data['itunes_artist_id'] = artist_id_for_genres or ''
if self.database.add_to_discovery_pool(track_data, source=db_source, profile_id=profile_id): if self.database.add_to_discovery_pool(track_data, source=db_source, profile_id=profile_id):
total_tracks_added += 1 total_tracks_added += 1
except Exception as track_error: except Exception:
continue continue
time.sleep(DELAY_BETWEEN_ALBUMS) time.sleep(DELAY_BETWEEN_ALBUMS)
@ -2819,6 +2837,7 @@ class WatchlistScanner:
album_type='album,single,ep', album_type='album,single,ep',
limit=5, limit=5,
skip_cache=True, skip_cache=True,
max_pages=1,
) )
if not recent_releases: if not recent_releases:
@ -3060,6 +3079,7 @@ class WatchlistScanner:
album_type='album,single,ep', album_type='album,single,ep',
limit=20, limit=20,
skip_cache=True, skip_cache=True,
max_pages=2,
) )
for album in albums or []: for album in albums or []:
process_album(album, artist.artist_name, artist.spotify_artist_id, fallback_id if fallback_source == 'itunes' else None, 'spotify') process_album(album, artist.artist_name, artist.spotify_artist_id, fallback_id if fallback_source == 'itunes' else None, 'spotify')
@ -3117,6 +3137,7 @@ class WatchlistScanner:
album_type='album,single,ep', album_type='album,single,ep',
limit=20, limit=20,
skip_cache=True, skip_cache=True,
max_pages=2,
) )
for album in albums or []: for album in albums or []:
process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, fallback_id if fallback_source == 'itunes' else None, 'spotify') process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, fallback_id if fallback_source == 'itunes' else None, 'spotify')

335
tests/test_watchlist_scanner_scan.py
View file

@ -76,11 +76,14 @@ class _FakeMetadataService:
class _FakeSourceClient: class _FakeSourceClient:
def __init__(self, *, artist_id: str, albums, image_url: str): def __init__(self, *, artist_id: str, albums, image_url: str, album_payload=None, album_search_results=None):
self.artist_id = artist_id self.artist_id = artist_id
self.albums = list(albums) self.albums = list(albums)
self.image_url = image_url self.image_url = image_url
self.album_payload = album_payload
self.album_search_results = list(album_search_results or [])
self.search_calls = [] self.search_calls = []
self.search_album_calls = []
self.album_calls = [] self.album_calls = []
self.artist_calls = [] self.artist_calls = []
@ -88,22 +91,41 @@ class _FakeSourceClient:
self.search_calls.append((query, limit, kwargs)) self.search_calls.append((query, limit, kwargs))
return [types.SimpleNamespace(id=self.artist_id, name=query)] return [types.SimpleNamespace(id=self.artist_id, name=query)]
def search_albums(self, query, limit=1, **kwargs):
self.search_album_calls.append((query, limit, kwargs))
return list(self.album_search_results)
def get_artist_albums(self, artist_id, album_type='album,single', limit=50, **kwargs): def get_artist_albums(self, artist_id, album_type='album,single', limit=50, **kwargs):
self.album_calls.append((artist_id, album_type, limit, kwargs)) self.album_calls.append((artist_id, album_type, limit, kwargs))
return list(self.albums) return list(self.albums)
def get_artist(self, artist_id): def get_artist(self, artist_id, **kwargs):
self.artist_calls.append(artist_id) self.artist_calls.append(artist_id)
return { return {
"id": artist_id, "id": artist_id,
"images": [{"url": self.image_url}] if self.image_url else [], "images": [{"url": self.image_url}] if self.image_url else [],
} }
def get_album(self, album_id, **kwargs):
self.album_calls.append((album_id, kwargs))
if self.album_payload is not None:
return self.album_payload
return {
"id": album_id,
"name": "Album One",
"images": [{"url": self.image_url}] if self.image_url else [],
"tracks": {"items": []},
"artists": [{"id": self.artist_id}],
}
class _FakeDB: class _FakeDB:
def __init__(self, artists): def __init__(self, artists):
self.artists = artists self.artists = artists
self.similar_calls = [] self.similar_calls = []
self.discovery_pool_calls = []
self.discovery_pool_timestamp_calls = []
self.db_albums = []
def get_watchlist_artists(self, profile_id=None): def get_watchlist_artists(self, profile_id=None):
return list(self.artists) return list(self.artists)
@ -112,6 +134,52 @@ class _FakeDB:
self.similar_calls.append((args, kwargs)) self.similar_calls.append((args, kwargs))
return False return False
def should_populate_discovery_pool(self, hours_threshold=24, profile_id=1):
return True
def get_top_similar_artists(self, limit=50, profile_id=1):
return []
def add_to_discovery_pool(self, track_data, source, profile_id=1):
self.discovery_pool_calls.append((track_data, source, profile_id))
return True
def cleanup_old_discovery_tracks(self, days_threshold=365):
return 0
def update_discovery_pool_timestamp(self, track_count, profile_id=1):
self.discovery_pool_timestamp_calls.append((track_count, profile_id))
return True
class _Cursor:
def __init__(self, parent):
self.parent = parent
def execute(self, *args, **kwargs):
return None
def fetchall(self):
return list(self.parent.db_albums)
def fetchone(self):
return {"count": 0}
class _Conn:
def __init__(self, cursor):
self._cursor = cursor
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def cursor(self):
return self._cursor
def _get_connection(self):
return self._Conn(self._Cursor(self))
def _build_artist(name="Artist One", profile_id=11): def _build_artist(name="Artist One", profile_id=11):
return types.SimpleNamespace( return types.SimpleNamespace(
@ -384,6 +452,269 @@ def test_get_artist_discography_for_watchlist_falls_back_when_primary_empty(monk
assert spotify_client.album_calls assert spotify_client.album_calls
def test_populate_discovery_pool_uses_primary_source_first(monkeypatch):
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0)
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0)
monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None))
monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
similar_artist = types.SimpleNamespace(
id=501,
similar_artist_name="Similar Artist",
occurrence_count=3,
similar_artist_spotify_id="sp-artist",
similar_artist_itunes_id="it-artist",
similar_artist_deezer_id="dz-artist",
)
album = types.SimpleNamespace(id="dz-album-1", name="Deezer Album", album_type="album")
deezer_album_payload = {
"id": "dz-album-1",
"name": "Deezer Album",
"images": [{"url": "https://example.com/deezer-album.jpg"}],
"release_date": "2026-04-01",
"popularity": 0,
"tracks": {
"items": [
{
"id": "dz-track-1",
"name": "Deezer Track",
"duration_ms": 123456,
"artists": [{"name": "Similar Artist"}],
}
]
},
"artists": [{"id": "dz-artist"}],
}
deezer_client = _FakeSourceClient(
artist_id="dz-artist",
albums=[album],
image_url="https://example.com/deezer-artist.jpg",
album_payload=deezer_album_payload,
)
spotify_client = _FakeSourceClient(
artist_id="sp-artist",
albums=[types.SimpleNamespace(id="sp-album-1", name="Spotify Album", album_type="album")],
image_url="https://example.com/spotify-artist.jpg",
album_payload={
"id": "sp-album-1",
"name": "Spotify Album",
"images": [{"url": "https://example.com/spotify-album.jpg"}],
"release_date": "2026-04-01",
"popularity": 50,
"tracks": {"items": []},
"artists": [{"id": "sp-artist"}],
},
)
def fake_get_client_for_source(source):
return {
"deezer": deezer_client,
"spotify": spotify_client,
}.get(source)
monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source)
scanner = _build_scanner({"tracks": {"items": []}}, [])
scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False
scanner.database.should_populate_discovery_pool = lambda hours_threshold=24, profile_id=1: True
scanner.database.get_top_similar_artists = lambda limit=50, profile_id=1: [similar_artist]
scanner.database.db_albums = []
scanner.cache_discovery_recent_albums = lambda *args, **kwargs: None
scanner.curate_discovery_playlists = lambda *args, **kwargs: None
scanner.database.update_discovery_pool_timestamp = lambda *args, **kwargs: True
scanner.database.cleanup_old_discovery_tracks = lambda *args, **kwargs: 0
scanner.populate_discovery_pool(top_artists_limit=1, albums_per_artist=1, profile_id=1)
assert scanner.database.discovery_pool_calls
assert scanner.database.discovery_pool_calls[0][1] == "deezer"
assert deezer_client.album_calls
assert spotify_client.search_calls == []
assert spotify_client.artist_calls == []
def test_populate_discovery_pool_falls_back_to_spotify_when_primary_has_no_albums(monkeypatch):
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0)
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0)
monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None))
monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
similar_artist = types.SimpleNamespace(
id=502,
similar_artist_name="Fallback Artist",
occurrence_count=1,
similar_artist_spotify_id="sp-artist",
similar_artist_itunes_id="it-artist",
similar_artist_deezer_id="dz-artist",
)
deezer_client = _FakeSourceClient(
artist_id="dz-artist",
albums=[],
image_url="https://example.com/deezer-artist.jpg",
)
spotify_album = types.SimpleNamespace(id="sp-album-1", name="Spotify Album", album_type="album")
spotify_client = _FakeSourceClient(
artist_id="sp-artist",
albums=[spotify_album],
image_url="https://example.com/spotify-artist.jpg",
album_payload={
"id": "sp-album-1",
"name": "Spotify Album",
"images": [{"url": "https://example.com/spotify-album.jpg"}],
"release_date": "2026-04-01",
"popularity": 50,
"tracks": {
"items": [
{
"id": "sp-track-1",
"name": "Spotify Track",
"duration_ms": 234567,
"artists": [{"name": "Fallback Artist"}],
}
]
},
"artists": [{"id": "sp-artist"}],
},
)
def fake_get_client_for_source(source):
return {
"deezer": deezer_client,
"spotify": spotify_client,
}.get(source)
monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source)
scanner = _build_scanner({"tracks": {"items": []}}, [])
scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False
scanner.database.should_populate_discovery_pool = lambda hours_threshold=24, profile_id=1: True
scanner.database.get_top_similar_artists = lambda limit=50, profile_id=1: [similar_artist]
scanner.database.db_albums = []
scanner.cache_discovery_recent_albums = lambda *args, **kwargs: None
scanner.curate_discovery_playlists = lambda *args, **kwargs: None
scanner.database.update_discovery_pool_timestamp = lambda *args, **kwargs: True
scanner.database.cleanup_old_discovery_tracks = lambda *args, **kwargs: 0
scanner.populate_discovery_pool(top_artists_limit=1, albums_per_artist=1, profile_id=1)
assert scanner.database.discovery_pool_calls
assert scanner.database.discovery_pool_calls[0][1] == "spotify"
assert deezer_client.album_calls
assert spotify_client.search_calls == [("Fallback Artist", 1, {"allow_fallback": False})]
assert spotify_client.album_calls
assert any(
isinstance(call, tuple)
and len(call) == 4
and call[0] == "sp-artist"
and call[3].get("skip_cache") is False
and call[3].get("allow_fallback") is False
and call[3].get("max_pages") == 2
for call in spotify_client.album_calls
)
assert any(
isinstance(call, tuple)
and len(call) == 4
and call[3].get("allow_fallback") is False
for call in spotify_client.album_calls
)
def test_populate_discovery_pool_uses_strict_spotify_for_database_album_search(monkeypatch):
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ARTISTS", 0)
monkeypatch.setattr(watchlist_scanner_module, "DELAY_BETWEEN_ALBUMS", 0)
monkeypatch.setattr(watchlist_scanner_module, "time", types.SimpleNamespace(sleep=lambda *_args, **_kwargs: None))
monkeypatch.setattr(watchlist_scanner_module, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(watchlist_scanner_module, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
similar_artist = types.SimpleNamespace(
id=503,
similar_artist_name="No Album Artist",
occurrence_count=1,
similar_artist_spotify_id="sp-artist",
similar_artist_itunes_id="it-artist",
similar_artist_deezer_id="dz-artist",
)
deezer_client = _FakeSourceClient(
artist_id="dz-artist",
albums=[],
image_url="https://example.com/deezer-artist.jpg",
)
spotify_client = _FakeSourceClient(
artist_id="sp-artist",
albums=[],
image_url="https://example.com/spotify-artist.jpg",
album_search_results=[types.SimpleNamespace(id="sp-db-album", name="DB Album")],
album_payload={
"id": "sp-db-album",
"name": "DB Album",
"images": [{"url": "https://example.com/db-album.jpg"}],
"release_date": "2026-04-01",
"popularity": 75,
"tracks": {
"items": [
{
"id": "sp-db-track-1",
"name": "DB Track",
"duration_ms": 345678,
"artists": [{"name": "DB Artist"}],
}
]
},
"artists": [{"id": "sp-artist"}],
},
)
def fake_get_client_for_source(source):
return {
"deezer": deezer_client,
"spotify": spotify_client,
}.get(source)
monkeypatch.setattr(watchlist_scanner_module, "get_client_for_source", fake_get_client_for_source)
scanner = _build_scanner({"tracks": {"items": []}}, [])
scanner._database.has_fresh_similar_artists = lambda *args, **kwargs: False
scanner.database.should_populate_discovery_pool = lambda hours_threshold=24, profile_id=1: True
scanner.database.get_top_similar_artists = lambda limit=50, profile_id=1: [similar_artist]
scanner.database.db_albums = [{"title": "DB Album", "artist_name": "DB Artist"}]
scanner.cache_discovery_recent_albums = lambda *args, **kwargs: None
scanner.curate_discovery_playlists = lambda *args, **kwargs: None
scanner.database.update_discovery_pool_timestamp = lambda *args, **kwargs: True
scanner.database.cleanup_old_discovery_tracks = lambda *args, **kwargs: 0
scanner.populate_discovery_pool(top_artists_limit=1, albums_per_artist=1, profile_id=1)
assert scanner.database.discovery_pool_calls
assert scanner.database.discovery_pool_calls[0][1] == "spotify"
assert spotify_client.search_album_calls
assert any(
kwargs.get("allow_fallback") is False
for _, _, kwargs in spotify_client.search_album_calls
)
assert any(
isinstance(call, tuple)
and len(call) == 4
and call[0] == "sp-artist"
and call[3].get("skip_cache") is False
and call[3].get("allow_fallback") is False
and call[3].get("max_pages") == 2
for call in spotify_client.album_calls
)
assert any(
isinstance(call, tuple)
and len(call) == 2
and call[1].get("allow_fallback") is False
for call in spotify_client.album_calls
if len(call) == 2
)
def test_match_to_spotify_uses_strict_lookup(): def test_match_to_spotify_uses_strict_lookup():
spotify_client = _FakeSpotifyClient( spotify_client = _FakeSpotifyClient(
search_results=[types.SimpleNamespace(id="fallback-id", name="Artist One")] search_results=[types.SimpleNamespace(id="fallback-id", name="Artist One")]