f6b0bd30e3
- Was only backfilling the active provider — artists added via Deezer never got Spotify/iTunes IDs, and vice versa - Now backfills iTunes (always), Deezer (always), and Spotify (if authenticated) at the start of every scan - Added _match_to_deezer() and update_watchlist_deezer_id() for Deezer cross-provider matching - Generalized backfill with provider→attribute/function maps
3208 lines
No EOL
163 KiB
Python
3208 lines
No EOL
163 KiB
Python
#!/usr/bin/env python3
|
|
|
|
"""
|
|
Watchlist Scanner Service - Monitors watched artists for new releases
|
|
"""
|
|
|
|
from typing import List, Dict, Any, Optional
|
|
from datetime import datetime, timezone, timedelta
|
|
from dataclasses import dataclass
|
|
import re
|
|
import time
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from database.music_database import get_database, WatchlistArtist
|
|
from core.spotify_client import SpotifyClient
|
|
from core.wishlist_service import get_wishlist_service
|
|
from core.matching_engine import MusicMatchingEngine
|
|
from utils.logging_config import get_logger
|
|
|
|
logger = get_logger("watchlist_scanner")
|
|
|
|
# Rate limiting constants for watchlist operations
|
|
DELAY_BETWEEN_ARTISTS = 4.0 # 4 seconds between different artists (was 2s, increased to reduce Spotify rate limit risk)
|
|
DELAY_BETWEEN_ALBUMS = 0.5 # 500ms between albums for same artist
|
|
DELAY_BETWEEN_API_BATCHES = 1.0 # 1 second between API batch operations
|
|
|
|
# iTunes API retry configuration
|
|
ITUNES_MAX_RETRIES = 3
|
|
ITUNES_BASE_DELAY = 1.0 # Base delay in seconds for exponential backoff
|
|
|
|
|
|
def _get_fallback_metadata_client():
|
|
"""Get the configured metadata fallback client (iTunes or Deezer)."""
|
|
try:
|
|
from config.settings import config_manager
|
|
source = config_manager.get('metadata.fallback_source', 'itunes') or 'itunes'
|
|
if source == 'deezer':
|
|
from core.deezer_client import DeezerClient
|
|
return DeezerClient(), 'deezer'
|
|
from core.itunes_client import iTunesClient
|
|
return iTunesClient(), 'itunes'
|
|
except Exception:
|
|
from core.itunes_client import iTunesClient
|
|
return iTunesClient(), 'itunes'
|
|
|
|
|
|
def itunes_api_call_with_retry(func, *args, max_retries=ITUNES_MAX_RETRIES, **kwargs):
|
|
"""
|
|
Execute an iTunes API call with exponential backoff retry logic.
|
|
|
|
Args:
|
|
func: The function to call
|
|
*args: Arguments to pass to the function
|
|
max_retries: Maximum number of retry attempts
|
|
**kwargs: Keyword arguments to pass to the function
|
|
|
|
Returns:
|
|
The result of the function call, or None if all retries failed
|
|
"""
|
|
last_error = None
|
|
for attempt in range(max_retries):
|
|
try:
|
|
result = func(*args, **kwargs)
|
|
return result
|
|
except requests.exceptions.HTTPError as e:
|
|
# Handle rate limiting (429) and server errors (5xx)
|
|
if e.response is not None and e.response.status_code == 429:
|
|
delay = ITUNES_BASE_DELAY * (2 ** attempt)
|
|
logger.warning(f"[iTunes] Rate limited, retrying in {delay}s (attempt {attempt + 1}/{max_retries})")
|
|
time.sleep(delay)
|
|
last_error = e
|
|
elif e.response is not None and e.response.status_code >= 500:
|
|
delay = ITUNES_BASE_DELAY * (2 ** attempt)
|
|
logger.warning(f"[iTunes] Server error {e.response.status_code}, retrying in {delay}s (attempt {attempt + 1}/{max_retries})")
|
|
time.sleep(delay)
|
|
last_error = e
|
|
else:
|
|
raise # Don't retry on client errors (4xx except 429)
|
|
except requests.exceptions.RequestException as e:
|
|
# Retry on connection errors
|
|
delay = ITUNES_BASE_DELAY * (2 ** attempt)
|
|
logger.warning(f"[iTunes] Connection error, retrying in {delay}s (attempt {attempt + 1}/{max_retries}): {e}")
|
|
time.sleep(delay)
|
|
last_error = e
|
|
except Exception as e:
|
|
# Don't retry on other exceptions
|
|
raise
|
|
|
|
if last_error:
|
|
logger.error(f"[iTunes] All {max_retries} retry attempts failed: {last_error}")
|
|
return None
|
|
|
|
|
|
def clean_track_name_for_search(track_name):
|
|
"""
|
|
Intelligently cleans a track name for searching by removing noise while preserving important version information.
|
|
Removes: (feat. Artist), (Explicit), (Clean), etc.
|
|
Keeps: (Extended Version), (Live), (Acoustic), (Remix), etc.
|
|
"""
|
|
if not track_name or not isinstance(track_name, str):
|
|
return track_name
|
|
|
|
cleaned_name = track_name
|
|
|
|
# Define patterns to REMOVE (noise that doesn't affect track identity)
|
|
remove_patterns = [
|
|
r'\s*\(explicit\)', # (Explicit)
|
|
r'\s*\(clean\)', # (Clean)
|
|
r'\s*\(radio\s*edit\)', # (Radio Edit)
|
|
r'\s*\(radio\s*version\)', # (Radio Version)
|
|
r'\s*\(feat\.?\s*[^)]+\)', # (feat. Artist) or (ft. Artist)
|
|
r'\s*\(ft\.?\s*[^)]+\)', # (ft Artist)
|
|
r'\s*\(featuring\s*[^)]+\)', # (featuring Artist)
|
|
r'\s*\(with\s*[^)]+\)', # (with Artist)
|
|
r'\s*\[[^\]]*explicit[^\]]*\]', # [Explicit] in brackets
|
|
r'\s*\[[^\]]*clean[^\]]*\]', # [Clean] in brackets
|
|
]
|
|
|
|
# Apply removal patterns
|
|
for pattern in remove_patterns:
|
|
cleaned_name = re.sub(pattern, '', cleaned_name, flags=re.IGNORECASE).strip()
|
|
|
|
# PRESERVE important version information (do NOT remove these)
|
|
# These patterns are intentionally NOT in the remove list:
|
|
# - (Extended Version), (Extended), (Long Version)
|
|
# - (Live), (Live Version), (Concert)
|
|
# - (Acoustic), (Acoustic Version)
|
|
# - (Remix), (Club Mix), (Dance Mix)
|
|
# - (Remastered), (Remaster)
|
|
# - (Demo), (Studio Version)
|
|
# - (Instrumental)
|
|
# - Album/year info like (2023), (Deluxe Edition)
|
|
|
|
# If cleaning results in an empty string, return the original track name
|
|
if not cleaned_name.strip():
|
|
return track_name
|
|
|
|
# Log cleaning if significant changes were made
|
|
if cleaned_name != track_name:
|
|
logger.debug(f"🧹 Intelligent track cleaning: '{track_name}' -> '{cleaned_name}'")
|
|
|
|
return cleaned_name
|
|
|
|
def is_live_version(track_name: str, album_name: str = "") -> bool:
|
|
"""
|
|
Detect if a track or album is a live version.
|
|
|
|
Args:
|
|
track_name: Track name to check
|
|
album_name: Album name to check (optional)
|
|
|
|
Returns:
|
|
True if this is a live version, False otherwise
|
|
"""
|
|
if not track_name:
|
|
return False
|
|
|
|
# Combine track and album names for comprehensive checking
|
|
text_to_check = f"{track_name} {album_name}".lower()
|
|
|
|
# Live version patterns
|
|
live_patterns = [
|
|
r'\blive\b', # (Live), Live at, etc.
|
|
r'\blive at\b', # Live at Madison Square Garden
|
|
r'\bconcert\b', # Concert, Live Concert
|
|
r'\bin concert\b', # In Concert
|
|
r'\bunplugged\b', # MTV Unplugged (usually live)
|
|
r'\blive session\b', # Live Session
|
|
r'\blive from\b', # Live from...
|
|
r'\blive recording\b', # Live Recording
|
|
r'\bon stage\b', # On Stage
|
|
]
|
|
|
|
for pattern in live_patterns:
|
|
if re.search(pattern, text_to_check, re.IGNORECASE):
|
|
return True
|
|
|
|
return False
|
|
|
|
def is_remix_version(track_name: str, album_name: str = "") -> bool:
|
|
"""
|
|
Detect if a track is a remix.
|
|
|
|
Args:
|
|
track_name: Track name to check
|
|
album_name: Album name to check (optional)
|
|
|
|
Returns:
|
|
True if this is a remix, False otherwise
|
|
"""
|
|
if not track_name:
|
|
return False
|
|
|
|
# Combine track and album names for comprehensive checking
|
|
text_to_check = f"{track_name} {album_name}".lower()
|
|
|
|
# Remix patterns (but NOT remaster/remastered)
|
|
remix_patterns = [
|
|
r'\bremix\b', # Remix, Remixed
|
|
r'\bmix\b(?!.*\bremaster)', # Mix (but not if followed by remaster)
|
|
r'\bedit\b', # Radio Edit, Extended Edit
|
|
r'\bversion\b(?=.*\bmix\b)', # Version with Mix (e.g., "Dance Version Mix")
|
|
r'\bclub mix\b', # Club Mix
|
|
r'\bdance mix\b', # Dance Mix
|
|
r'\bradio edit\b', # Radio Edit
|
|
r'\bextended\b(?=.*\bmix\b)', # Extended Mix
|
|
r'\bdub\b', # Dub version
|
|
r'\bvip mix\b', # VIP Mix
|
|
]
|
|
|
|
# But exclude remaster/remastered - those are originals
|
|
if re.search(r'\bremaster(ed)?\b', text_to_check, re.IGNORECASE):
|
|
return False
|
|
|
|
for pattern in remix_patterns:
|
|
if re.search(pattern, text_to_check, re.IGNORECASE):
|
|
return True
|
|
|
|
return False
|
|
|
|
def is_acoustic_version(track_name: str, album_name: str = "") -> bool:
|
|
"""
|
|
Detect if a track is an acoustic version.
|
|
|
|
Args:
|
|
track_name: Track name to check
|
|
album_name: Album name to check (optional)
|
|
|
|
Returns:
|
|
True if this is an acoustic version, False otherwise
|
|
"""
|
|
if not track_name:
|
|
return False
|
|
|
|
# Combine track and album names for comprehensive checking
|
|
text_to_check = f"{track_name} {album_name}".lower()
|
|
|
|
# Acoustic version patterns
|
|
acoustic_patterns = [
|
|
r'\bacoustic\b', # Acoustic, Acoustic Version
|
|
r'\bstripped\b', # Stripped version
|
|
r'\bpiano version\b', # Piano Version
|
|
r'\bunplugged\b', # MTV Unplugged (can be acoustic)
|
|
]
|
|
|
|
for pattern in acoustic_patterns:
|
|
if re.search(pattern, text_to_check, re.IGNORECASE):
|
|
return True
|
|
|
|
return False
|
|
|
|
def is_instrumental_version(track_name: str, album_name: str = "") -> bool:
|
|
"""
|
|
Detect if a track is an instrumental version.
|
|
|
|
Args:
|
|
track_name: Track name to check
|
|
album_name: Album name to check (optional)
|
|
|
|
Returns:
|
|
True if this is an instrumental version, False otherwise
|
|
"""
|
|
if not track_name:
|
|
return False
|
|
|
|
text_to_check = f"{track_name} {album_name}".lower()
|
|
|
|
instrumental_patterns = [
|
|
r'\binstrumental\b', # Instrumental, Instrumental Version
|
|
r'\binst\.\b', # Inst. (abbreviation)
|
|
r'\bkaraoke\b', # Karaoke version
|
|
r'\bbacking track\b', # Backing Track
|
|
]
|
|
|
|
for pattern in instrumental_patterns:
|
|
if re.search(pattern, text_to_check, re.IGNORECASE):
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def matches_custom_exclude_terms(track_name: str, album_name: str, exclude_terms: list) -> str:
|
|
"""
|
|
Check if a track or album name contains any user-defined exclusion terms.
|
|
|
|
Args:
|
|
track_name: Track name to check
|
|
album_name: Album name to check
|
|
exclude_terms: List of terms to exclude (case-insensitive)
|
|
|
|
Returns:
|
|
The matched term if found, empty string if no match
|
|
"""
|
|
if not exclude_terms:
|
|
return ""
|
|
|
|
text_to_check = f"{track_name} {album_name}".lower()
|
|
|
|
for term in exclude_terms:
|
|
term = term.strip().lower()
|
|
if not term:
|
|
continue
|
|
if term in text_to_check:
|
|
return term
|
|
|
|
return ""
|
|
|
|
|
|
def is_compilation_album(album_name: str) -> bool:
|
|
"""
|
|
Detect if an album is a compilation/greatest hits album.
|
|
|
|
Args:
|
|
album_name: Album name to check
|
|
|
|
Returns:
|
|
True if this is a compilation album, False otherwise
|
|
"""
|
|
if not album_name:
|
|
return False
|
|
|
|
album_lower = album_name.lower()
|
|
|
|
# Compilation album patterns
|
|
compilation_patterns = [
|
|
r'\bgreatest hits\b', # Greatest Hits
|
|
r'\bbest of\b', # Best Of
|
|
r'\banthology\b', # Anthology
|
|
r'\bcollection\b', # Collection
|
|
r'\bcompilation\b', # Compilation
|
|
r'\bthe essential\b', # The Essential...
|
|
r'\bcomplete\b', # Complete Collection
|
|
r'\bhits\b', # Hits (standalone or at end)
|
|
r'\btop\s+\d+\b', # Top 10, Top 40, etc.
|
|
r'\bvery best\b', # Very Best Of
|
|
r'\bdefinitive\b', # Definitive Collection
|
|
]
|
|
|
|
for pattern in compilation_patterns:
|
|
if re.search(pattern, album_lower, re.IGNORECASE):
|
|
return True
|
|
|
|
return False
|
|
|
|
@dataclass
|
|
class ScanResult:
|
|
"""Result of scanning a single artist"""
|
|
artist_name: str
|
|
spotify_artist_id: str
|
|
albums_checked: int
|
|
new_tracks_found: int
|
|
tracks_added_to_wishlist: int
|
|
success: bool
|
|
error_message: Optional[str] = None
|
|
|
|
class WatchlistScanner:
|
|
"""Service for scanning watched artists for new releases"""
|
|
|
|
def __init__(self, spotify_client: SpotifyClient = None, metadata_service=None, database_path: str = "database/music_library.db"):
|
|
# Support both old (spotify_client) and new (metadata_service) initialization
|
|
self.database_path = database_path
|
|
self._database = None
|
|
self._wishlist_service = None
|
|
self._matching_engine = None
|
|
|
|
if metadata_service:
|
|
self._metadata_service = metadata_service
|
|
self.spotify_client = metadata_service.spotify # For backward compatibility
|
|
elif spotify_client:
|
|
self.spotify_client = spotify_client
|
|
self._metadata_service = None # Lazy load if needed
|
|
else:
|
|
raise ValueError("Must provide either spotify_client or metadata_service")
|
|
|
|
@property
|
|
def database(self):
|
|
"""Get database instance (lazy loading)"""
|
|
if self._database is None:
|
|
self._database = get_database(self.database_path)
|
|
return self._database
|
|
|
|
@property
|
|
def wishlist_service(self):
|
|
"""Get wishlist service instance (lazy loading)"""
|
|
if self._wishlist_service is None:
|
|
self._wishlist_service = get_wishlist_service()
|
|
return self._wishlist_service
|
|
|
|
@property
|
|
def matching_engine(self):
|
|
"""Get matching engine instance (lazy loading)"""
|
|
if self._matching_engine is None:
|
|
self._matching_engine = MusicMatchingEngine()
|
|
return self._matching_engine
|
|
|
|
@property
|
|
def metadata_service(self):
|
|
"""Get or create MetadataService instance (lazy loading)"""
|
|
if self._metadata_service is None:
|
|
from core.metadata_service import MetadataService
|
|
self._metadata_service = MetadataService()
|
|
return self._metadata_service
|
|
|
|
def _get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist):
|
|
"""
|
|
Get the appropriate client and artist ID based on active provider.
|
|
If iTunes ID is missing, searches by artist name to find and cache it.
|
|
|
|
Returns:
|
|
Tuple of (client, artist_id, provider_name) or (None, None, None) if no valid ID
|
|
"""
|
|
provider = self.metadata_service.get_active_provider()
|
|
|
|
if provider == 'spotify':
|
|
if watchlist_artist.spotify_artist_id:
|
|
return (self.metadata_service.spotify, watchlist_artist.spotify_artist_id, 'spotify')
|
|
else:
|
|
logger.warning(f"No Spotify ID for {watchlist_artist.artist_name}, cannot scan with Spotify")
|
|
return (None, None, None)
|
|
else: # itunes or deezer fallback
|
|
fallback_source = provider # 'itunes' or 'deezer'
|
|
fallback_client = self.metadata_service.itunes # May be iTunesClient or DeezerClient
|
|
# Pick the right stored ID for the active fallback source
|
|
stored_id = watchlist_artist.deezer_artist_id if fallback_source == 'deezer' else watchlist_artist.itunes_artist_id
|
|
if stored_id:
|
|
return (fallback_client, stored_id, fallback_source)
|
|
else:
|
|
# No ID stored for this source - search by name and cache it
|
|
logger.info(f"No {fallback_source} ID for {watchlist_artist.artist_name}, searching by name...")
|
|
try:
|
|
search_results = fallback_client.search_artists(watchlist_artist.artist_name, limit=1)
|
|
if search_results and len(search_results) > 0:
|
|
found_id = search_results[0].id
|
|
logger.info(f"Found {fallback_source} ID {found_id} for {watchlist_artist.artist_name}")
|
|
# Cache the ID in the database for future use
|
|
if fallback_source == 'deezer':
|
|
self.database.update_watchlist_artist_deezer_id(
|
|
watchlist_artist.spotify_artist_id or str(watchlist_artist.id),
|
|
found_id
|
|
)
|
|
else:
|
|
self.database.update_watchlist_artist_itunes_id(
|
|
watchlist_artist.spotify_artist_id or str(watchlist_artist.id),
|
|
found_id
|
|
)
|
|
return (fallback_client, found_id, fallback_source)
|
|
else:
|
|
logger.warning(f"Could not find {watchlist_artist.artist_name} on {fallback_source}")
|
|
return (None, None, None)
|
|
except Exception as e:
|
|
logger.error(f"Error searching {fallback_source} for {watchlist_artist.artist_name}: {e}")
|
|
return (None, None, None)
|
|
|
|
def get_active_client_and_artist_id(self, watchlist_artist: WatchlistArtist):
|
|
"""
|
|
Public wrapper for _get_active_client_and_artist_id.
|
|
Gets the appropriate client and artist ID based on active provider.
|
|
|
|
Returns:
|
|
Tuple of (client, artist_id, provider_name) or (None, None, None) if no valid ID
|
|
"""
|
|
return self._get_active_client_and_artist_id(watchlist_artist)
|
|
|
|
def get_artist_image_url(self, watchlist_artist: WatchlistArtist) -> Optional[str]:
|
|
"""
|
|
Get artist image URL using the active provider.
|
|
|
|
Returns:
|
|
Image URL string or None if not available
|
|
"""
|
|
client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist)
|
|
if not client or not artist_id:
|
|
return None
|
|
|
|
try:
|
|
artist_data = client.get_artist(artist_id)
|
|
if artist_data:
|
|
# Handle both Spotify and iTunes response formats
|
|
if 'images' in artist_data and artist_data['images']:
|
|
return artist_data['images'][0].get('url')
|
|
elif 'image_url' in artist_data:
|
|
return artist_data['image_url']
|
|
except Exception as e:
|
|
logger.debug(f"Could not fetch artist image for {watchlist_artist.artist_name}: {e}")
|
|
|
|
return None
|
|
|
|
def get_artist_discography_for_watchlist(self, watchlist_artist: WatchlistArtist, last_scan_timestamp: Optional[datetime] = None) -> Optional[List]:
|
|
"""
|
|
Get artist's discography using the active provider, with proper ID resolution.
|
|
This is the provider-aware version of get_artist_discography.
|
|
|
|
Args:
|
|
watchlist_artist: WatchlistArtist object (has both spotify and itunes IDs)
|
|
last_scan_timestamp: Only return releases after this date (for incremental scans)
|
|
|
|
Returns:
|
|
List of albums or None on error
|
|
"""
|
|
client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist)
|
|
if not client or not artist_id:
|
|
logger.warning(f"No valid client/ID for {watchlist_artist.artist_name}")
|
|
return None
|
|
|
|
albums = self._get_artist_discography_with_client(client, artist_id, last_scan_timestamp, lookback_days=watchlist_artist.lookback_days)
|
|
|
|
# If primary provider returned nothing, try the other provider as fallback
|
|
if not albums:
|
|
fallback_id = None
|
|
fallback_client = None
|
|
|
|
if provider == 'spotify':
|
|
fallback_client = self.metadata_service.itunes
|
|
fallback_id = watchlist_artist.itunes_artist_id
|
|
# If no iTunes ID stored, search by name and cache it
|
|
if not fallback_id:
|
|
try:
|
|
search_results = fallback_client.search_artists(watchlist_artist.artist_name, limit=1)
|
|
if search_results:
|
|
fallback_id = search_results[0].id
|
|
logger.info(f"Resolved iTunes ID {fallback_id} for {watchlist_artist.artist_name}")
|
|
self.database.update_watchlist_artist_itunes_id(
|
|
watchlist_artist.spotify_artist_id or str(watchlist_artist.id),
|
|
fallback_id
|
|
)
|
|
watchlist_artist.itunes_artist_id = fallback_id
|
|
except Exception as e:
|
|
logger.debug(f"Could not resolve iTunes ID for {watchlist_artist.artist_name}: {e}")
|
|
|
|
elif provider == 'itunes':
|
|
fallback_client = self.metadata_service.spotify
|
|
fallback_id = watchlist_artist.spotify_artist_id
|
|
|
|
if fallback_client and fallback_id:
|
|
logger.info(f"{provider.capitalize()} returned no albums for {watchlist_artist.artist_name}, falling back to {'iTunes' if provider == 'spotify' else 'Spotify'}")
|
|
albums = self._get_artist_discography_with_client(fallback_client, fallback_id, last_scan_timestamp, lookback_days=watchlist_artist.lookback_days)
|
|
|
|
return albums
|
|
|
|
def scan_all_watchlist_artists(self) -> List[ScanResult]:
|
|
"""
|
|
Scan artists in the watchlist for new releases.
|
|
|
|
OPTIMIZED: Scans up to 50 artists per run using smart selection:
|
|
- Priority: Artists not scanned in 7+ days (guaranteed)
|
|
- Remainder: Random selection from other artists
|
|
|
|
This reduces API calls while ensuring all artists scanned at least weekly.
|
|
Only checks releases after their last scan timestamp.
|
|
"""
|
|
logger.info("Starting watchlist scan")
|
|
|
|
try:
|
|
from datetime import datetime, timedelta
|
|
import random
|
|
|
|
# Get all watchlist artists
|
|
all_watchlist_artists = self.database.get_watchlist_artists()
|
|
if not all_watchlist_artists:
|
|
logger.info("No artists in watchlist to scan")
|
|
return []
|
|
|
|
logger.info(f"Found {len(all_watchlist_artists)} total artists in watchlist")
|
|
|
|
# OPTIMIZATION: Select up to 50 artists to scan
|
|
# 1. Must scan: Artists not scanned in 7+ days (or never scanned)
|
|
seven_days_ago = datetime.now() - timedelta(days=7)
|
|
must_scan = []
|
|
can_skip = []
|
|
|
|
for artist in all_watchlist_artists:
|
|
if artist.last_scan_timestamp is None:
|
|
# Never scanned - must scan
|
|
must_scan.append(artist)
|
|
elif artist.last_scan_timestamp < seven_days_ago:
|
|
# Not scanned in 7+ days - must scan
|
|
must_scan.append(artist)
|
|
else:
|
|
# Scanned recently - can skip (but might randomly select)
|
|
can_skip.append(artist)
|
|
|
|
logger.info(f"Artists requiring scan (not scanned in 7+ days): {len(must_scan)}")
|
|
logger.info(f"Artists scanned recently (< 7 days): {len(can_skip)}")
|
|
|
|
# 2. Fill remaining slots (up to 50 total) with random selection
|
|
max_artists_per_scan = 50
|
|
artists_to_scan = must_scan.copy()
|
|
|
|
remaining_slots = max_artists_per_scan - len(must_scan)
|
|
if remaining_slots > 0 and can_skip:
|
|
# Randomly sample from recently-scanned artists
|
|
random_sample_size = min(remaining_slots, len(can_skip))
|
|
random_selection = random.sample(can_skip, random_sample_size)
|
|
artists_to_scan.extend(random_selection)
|
|
logger.info(f"Additionally scanning {len(random_selection)} randomly selected artists")
|
|
|
|
# Shuffle to avoid always scanning same order
|
|
random.shuffle(artists_to_scan)
|
|
|
|
logger.info(f"Total artists to scan this run: {len(artists_to_scan)}")
|
|
if len(all_watchlist_artists) > max_artists_per_scan:
|
|
logger.info(f"Skipping {len(all_watchlist_artists) - len(artists_to_scan)} artists (will be scanned in future runs)")
|
|
|
|
watchlist_artists = artists_to_scan
|
|
|
|
# PROACTIVE ID BACKFILLING (cross-provider support)
|
|
# Before scanning, ensure ALL artists have IDs for ALL available sources
|
|
# iTunes and Deezer are always available; Spotify requires authentication
|
|
providers_to_backfill = ['itunes', 'deezer']
|
|
if self.spotify_client and self.spotify_client.is_spotify_authenticated():
|
|
providers_to_backfill.append('spotify')
|
|
|
|
for provider in providers_to_backfill:
|
|
try:
|
|
self._backfill_missing_ids(all_watchlist_artists, provider)
|
|
except Exception as backfill_error:
|
|
logger.warning(f"Error during {provider} ID backfilling: {backfill_error}")
|
|
# Continue with scan even if backfilling fails
|
|
|
|
scan_results = []
|
|
for i, artist in enumerate(watchlist_artists):
|
|
# Abort scan if Spotify is rate limited — don't keep hammering
|
|
if self.spotify_client and hasattr(self.spotify_client, 'is_rate_limited') and self.spotify_client.is_rate_limited():
|
|
logger.warning(f"⚠️ Spotify rate limited — aborting watchlist scan after {i}/{len(watchlist_artists)} artists")
|
|
break
|
|
|
|
try:
|
|
result = self.scan_artist(artist)
|
|
scan_results.append(result)
|
|
|
|
if result.success:
|
|
logger.info(f"✅ Scanned {artist.artist_name}: {result.new_tracks_found} new tracks found")
|
|
else:
|
|
logger.warning(f"❌ Failed to scan {artist.artist_name}: {result.error_message}")
|
|
|
|
# Rate limiting: Add delay between artists to avoid hitting Spotify API limits
|
|
# This is critical to prevent getting banned for 6+ hours
|
|
if i < len(watchlist_artists) - 1: # Don't delay after the last artist
|
|
logger.debug(f"Rate limiting: waiting {DELAY_BETWEEN_ARTISTS}s before scanning next artist")
|
|
time.sleep(DELAY_BETWEEN_ARTISTS)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error scanning artist {artist.artist_name}: {e}")
|
|
scan_results.append(ScanResult(
|
|
artist_name=artist.artist_name,
|
|
spotify_artist_id=artist.spotify_artist_id,
|
|
albums_checked=0,
|
|
new_tracks_found=0,
|
|
tracks_added_to_wishlist=0,
|
|
success=False,
|
|
error_message=str(e)
|
|
))
|
|
|
|
# Log summary
|
|
successful_scans = [r for r in scan_results if r.success]
|
|
total_new_tracks = sum(r.new_tracks_found for r in successful_scans)
|
|
total_added_to_wishlist = sum(r.tracks_added_to_wishlist for r in successful_scans)
|
|
|
|
logger.info(f"Watchlist scan complete: {len(successful_scans)}/{len(scan_results)} artists scanned successfully")
|
|
logger.info(f"Found {total_new_tracks} new tracks, added {total_added_to_wishlist} to wishlist")
|
|
|
|
# Populate discovery pool with tracks from similar artists
|
|
logger.info("Starting discovery pool population...")
|
|
self.populate_discovery_pool()
|
|
|
|
# Populate seasonal content (runs independently with its own threshold)
|
|
logger.info("Updating seasonal content...")
|
|
self._populate_seasonal_content()
|
|
|
|
# Sync Spotify library cache (runs after main scan)
|
|
try:
|
|
self.sync_spotify_library_cache()
|
|
except Exception as lib_err:
|
|
logger.warning(f"Error syncing Spotify library cache: {lib_err}")
|
|
|
|
return scan_results
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error during watchlist scan: {e}")
|
|
return []
|
|
|
|
def scan_artist(self, watchlist_artist: WatchlistArtist) -> ScanResult:
|
|
"""
|
|
Scan a single artist for new releases.
|
|
Only checks releases after the last scan timestamp.
|
|
Uses the active provider (Spotify if authenticated, otherwise iTunes).
|
|
"""
|
|
try:
|
|
logger.info(f"Scanning artist: {watchlist_artist.artist_name}")
|
|
|
|
# Get the active client and artist ID based on provider
|
|
client, artist_id, provider = self._get_active_client_and_artist_id(watchlist_artist)
|
|
|
|
if client is None or artist_id is None:
|
|
return ScanResult(
|
|
artist_name=watchlist_artist.artist_name,
|
|
spotify_artist_id=watchlist_artist.spotify_artist_id or '',
|
|
albums_checked=0,
|
|
new_tracks_found=0,
|
|
tracks_added_to_wishlist=0,
|
|
success=False,
|
|
error_message=f"No {self.metadata_service.get_active_provider()} ID available for this artist"
|
|
)
|
|
|
|
logger.info(f"Using {provider} provider for {watchlist_artist.artist_name} (ID: {artist_id})")
|
|
|
|
# Update artist image if missing or on every scan to keep fresh
|
|
try:
|
|
image_url = None
|
|
artist_data = client.get_artist(artist_id)
|
|
if artist_data:
|
|
if 'images' in artist_data and artist_data['images']:
|
|
# Spotify/Deezer format: array of {url, height, width}
|
|
image_url = artist_data['images'][1]['url'] if len(artist_data['images']) > 1 else artist_data['images'][0]['url']
|
|
elif artist_data.get('image_url'):
|
|
# Direct image_url format (iTunes/some providers)
|
|
image_url = artist_data['image_url']
|
|
|
|
if image_url:
|
|
db_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or watchlist_artist.deezer_artist_id or artist_id
|
|
self.database.update_watchlist_artist_image(db_artist_id, image_url)
|
|
if not watchlist_artist.image_url:
|
|
logger.info(f"Backfilled artist image for {watchlist_artist.artist_name}")
|
|
else:
|
|
logger.debug(f"No image available for {watchlist_artist.artist_name} from {provider}")
|
|
except Exception as img_error:
|
|
logger.warning(f"Could not update artist image for {watchlist_artist.artist_name}: {img_error}")
|
|
|
|
# Get artist discography using active provider
|
|
albums = self._get_artist_discography_with_client(client, artist_id, watchlist_artist.last_scan_timestamp, lookback_days=watchlist_artist.lookback_days)
|
|
|
|
if albums is None:
|
|
return ScanResult(
|
|
artist_name=watchlist_artist.artist_name,
|
|
spotify_artist_id=watchlist_artist.spotify_artist_id or '',
|
|
albums_checked=0,
|
|
new_tracks_found=0,
|
|
tracks_added_to_wishlist=0,
|
|
success=False,
|
|
error_message=f"Failed to get artist discography from {provider}"
|
|
)
|
|
|
|
logger.info(f"Found {len(albums)} albums/singles to check for {watchlist_artist.artist_name}")
|
|
|
|
# Safety check: Limit number of albums to scan to prevent extremely long sessions
|
|
MAX_ALBUMS_PER_ARTIST = 50 # Reasonable limit to prevent API abuse
|
|
if len(albums) > MAX_ALBUMS_PER_ARTIST:
|
|
logger.warning(f"Artist {watchlist_artist.artist_name} has {len(albums)} albums, limiting to {MAX_ALBUMS_PER_ARTIST} most recent")
|
|
albums = albums[:MAX_ALBUMS_PER_ARTIST] # Most recent albums are first
|
|
|
|
# Check each album/single for missing tracks
|
|
new_tracks_found = 0
|
|
tracks_added_to_wishlist = 0
|
|
|
|
for album_index, album in enumerate(albums):
|
|
try:
|
|
# Get full album data
|
|
logger.info(f"Checking album {album_index + 1}/{len(albums)}: {album.name}")
|
|
album_data = client.get_album(album.id)
|
|
if not album_data:
|
|
continue
|
|
|
|
# Get album tracks (works for both Spotify and iTunes)
|
|
# Spotify's get_album() includes tracks, but we use get_album_tracks() for consistency
|
|
tracks_data = client.get_album_tracks(album.id)
|
|
if not tracks_data or not tracks_data.get('items'):
|
|
continue
|
|
|
|
tracks = tracks_data['items']
|
|
logger.debug(f"Checking album: {album_data.get('name', 'Unknown')} ({len(tracks)} tracks)")
|
|
|
|
# Check if user wants this type of release
|
|
if not self._should_include_release(len(tracks), watchlist_artist):
|
|
release_type = "album" if len(tracks) >= 7 else ("EP" if len(tracks) >= 4 else "single")
|
|
logger.debug(f"Skipping {release_type}: {album_data.get('name', 'Unknown')} - user preference")
|
|
continue
|
|
|
|
# Check each track
|
|
for track in tracks:
|
|
# Check content type filters (live, remix, acoustic, compilation)
|
|
if not self._should_include_track(track, album_data, watchlist_artist):
|
|
continue # Skip this track based on content type preferences
|
|
|
|
if self.is_track_missing_from_library(track, album_name=album_data.get('name')):
|
|
new_tracks_found += 1
|
|
|
|
# Add to wishlist
|
|
if self.add_track_to_wishlist(track, album_data, watchlist_artist):
|
|
tracks_added_to_wishlist += 1
|
|
|
|
# Rate limiting: Add delay between albums to prevent API abuse
|
|
# This is especially important for artists with many albums
|
|
if album_index < len(albums) - 1: # Don't delay after the last album
|
|
logger.debug(f"Rate limiting: waiting {DELAY_BETWEEN_ALBUMS}s before next album")
|
|
time.sleep(DELAY_BETWEEN_ALBUMS)
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error checking album {album.name}: {e}")
|
|
continue
|
|
|
|
# Update last scan timestamp for this artist
|
|
self.update_artist_scan_timestamp(watchlist_artist)
|
|
|
|
# Fetch and store similar artists for discovery feature (with caching to avoid over-polling)
|
|
# Similar artists are fetched from MusicMap (works with any source) and matched to both Spotify and iTunes
|
|
source_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or str(watchlist_artist.id)
|
|
try:
|
|
# Check if we have fresh similar artists cached (< 30 days old)
|
|
# If Spotify is authenticated, also require Spotify IDs to be present
|
|
spotify_authenticated = self.spotify_client and self.spotify_client.is_spotify_authenticated()
|
|
artist_profile_id = getattr(watchlist_artist, 'profile_id', 1)
|
|
if self.database.has_fresh_similar_artists(source_artist_id, days_threshold=30, require_spotify=spotify_authenticated, profile_id=artist_profile_id):
|
|
logger.info(f"Similar artists for {watchlist_artist.artist_name} are cached and fresh, skipping MusicMap fetch")
|
|
# Even if cached, backfill missing iTunes IDs (seamless dual-source support)
|
|
self._backfill_similar_artists_itunes_ids(source_artist_id, profile_id=artist_profile_id)
|
|
else:
|
|
logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}...")
|
|
self.update_similar_artists(watchlist_artist, profile_id=artist_profile_id)
|
|
logger.info(f"Similar artists updated for {watchlist_artist.artist_name}")
|
|
except Exception as similar_error:
|
|
logger.warning(f"Failed to update similar artists for {watchlist_artist.artist_name}: {similar_error}")
|
|
|
|
return ScanResult(
|
|
artist_name=watchlist_artist.artist_name,
|
|
spotify_artist_id=watchlist_artist.spotify_artist_id or '',
|
|
albums_checked=len(albums),
|
|
new_tracks_found=new_tracks_found,
|
|
tracks_added_to_wishlist=tracks_added_to_wishlist,
|
|
success=True
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error scanning artist {watchlist_artist.artist_name}: {e}")
|
|
return ScanResult(
|
|
artist_name=watchlist_artist.artist_name,
|
|
spotify_artist_id=watchlist_artist.spotify_artist_id or '',
|
|
albums_checked=0,
|
|
new_tracks_found=0,
|
|
tracks_added_to_wishlist=0,
|
|
success=False,
|
|
error_message=str(e)
|
|
)
|
|
|
|
def get_artist_discography(self, spotify_artist_id: str, last_scan_timestamp: Optional[datetime] = None) -> Optional[List]:
|
|
"""
|
|
Get artist's discography from Spotify, optionally filtered by release date.
|
|
|
|
Args:
|
|
spotify_artist_id: Spotify artist ID
|
|
last_scan_timestamp: Only return releases after this date (for incremental scans)
|
|
If None, uses lookback period setting from database
|
|
"""
|
|
try:
|
|
# Get all artist albums (albums + singles) - this is rate limited in spotify_client
|
|
logger.debug(f"Fetching discography for artist {spotify_artist_id}")
|
|
albums = self.spotify_client.get_artist_albums(spotify_artist_id, album_type='album,single', limit=50, skip_cache=True)
|
|
|
|
if not albums:
|
|
logger.warning(f"No albums found for artist {spotify_artist_id}")
|
|
return []
|
|
|
|
# Add small delay after fetching artist discography to be extra safe
|
|
time.sleep(0.3) # 300ms breathing room
|
|
|
|
# Determine cutoff date for filtering
|
|
cutoff_timestamp = last_scan_timestamp
|
|
|
|
# If no last scan timestamp, use per-artist lookback or global setting
|
|
if cutoff_timestamp is None:
|
|
if lookback_days is not None:
|
|
# Per-artist override
|
|
cutoff_timestamp = datetime.now(timezone.utc) - timedelta(days=lookback_days)
|
|
logger.info(f"Using per-artist lookback: {lookback_days} days (cutoff: {cutoff_timestamp})")
|
|
else:
|
|
# Global setting
|
|
lookback_period = self._get_lookback_period_setting()
|
|
if lookback_period != 'all':
|
|
days = int(lookback_period)
|
|
cutoff_timestamp = datetime.now(timezone.utc) - timedelta(days=days)
|
|
logger.info(f"Using global lookback period: {lookback_period} days (cutoff: {cutoff_timestamp})")
|
|
|
|
# Filter by release date if we have a cutoff timestamp
|
|
if cutoff_timestamp:
|
|
filtered_albums = []
|
|
for album in albums:
|
|
if self.is_album_after_timestamp(album, cutoff_timestamp):
|
|
filtered_albums.append(album)
|
|
|
|
logger.info(f"Filtered {len(albums)} albums to {len(filtered_albums)} released after {cutoff_timestamp}")
|
|
return filtered_albums
|
|
|
|
# Return all albums if no cutoff (lookback_period = 'all')
|
|
return albums
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting discography for artist {spotify_artist_id}: {e}")
|
|
return None
|
|
|
|
def _get_artist_discography_with_client(self, client, artist_id: str, last_scan_timestamp: Optional[datetime] = None, lookback_days: Optional[int] = None) -> Optional[List]:
|
|
"""
|
|
Get artist's discography using the specified client, optionally filtered by release date.
|
|
|
|
Args:
|
|
client: The metadata client to use (spotify or itunes)
|
|
artist_id: Artist ID for the given client
|
|
last_scan_timestamp: Only return releases after this date (for incremental scans)
|
|
If None, uses lookback period setting from database
|
|
lookback_days: Per-artist override for lookback period (None = use global setting)
|
|
"""
|
|
try:
|
|
# Get all artist albums (albums + singles)
|
|
# skip_cache for Spotify so watchlist scans always get fresh data
|
|
logger.debug(f"Fetching discography for artist {artist_id}")
|
|
_skip = {'skip_cache': True} if hasattr(client, 'sp') else {}
|
|
albums = client.get_artist_albums(artist_id, album_type='album,single', limit=50, **_skip)
|
|
|
|
if not albums:
|
|
logger.warning(f"No albums found for artist {artist_id}")
|
|
return []
|
|
|
|
# Add small delay after fetching artist discography to be extra safe
|
|
time.sleep(0.3) # 300ms breathing room
|
|
|
|
# Determine cutoff date for filtering
|
|
lookback_period = self._get_lookback_period_setting()
|
|
|
|
# If lookback is 'all', always return everything regardless of scan timestamp
|
|
if lookback_period == 'all':
|
|
cutoff_timestamp = None
|
|
elif last_scan_timestamp is not None:
|
|
cutoff_timestamp = last_scan_timestamp
|
|
|
|
# Check if a lookback period change requires a one-time wider window
|
|
rescan_cutoff = self._get_rescan_cutoff()
|
|
if rescan_cutoff == 'all':
|
|
logger.info(f"Lookback period changed to 'all' — returning full discography")
|
|
cutoff_timestamp = None
|
|
elif rescan_cutoff is not None:
|
|
scan_ts = cutoff_timestamp
|
|
if scan_ts.tzinfo is None:
|
|
scan_ts = scan_ts.replace(tzinfo=timezone.utc)
|
|
if rescan_cutoff.tzinfo is None:
|
|
rescan_cutoff = rescan_cutoff.replace(tzinfo=timezone.utc)
|
|
if rescan_cutoff < scan_ts:
|
|
logger.info(f"Lookback period change detected — expanding cutoff from {cutoff_timestamp} to {rescan_cutoff}")
|
|
cutoff_timestamp = rescan_cutoff
|
|
else:
|
|
# No scan timestamp — use lookback period
|
|
days = int(lookback_period)
|
|
cutoff_timestamp = datetime.now(timezone.utc) - timedelta(days=days)
|
|
logger.info(f"Using lookback period: {lookback_period} days (cutoff: {cutoff_timestamp})")
|
|
|
|
# Filter by release date if we have a cutoff timestamp
|
|
if cutoff_timestamp:
|
|
filtered_albums = []
|
|
for album in albums:
|
|
if self.is_album_after_timestamp(album, cutoff_timestamp):
|
|
filtered_albums.append(album)
|
|
|
|
logger.info(f"Filtered {len(albums)} albums to {len(filtered_albums)} released after {cutoff_timestamp}")
|
|
return filtered_albums
|
|
|
|
# Return all albums if no cutoff (lookback_period = 'all')
|
|
return albums
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting discography for artist {artist_id}: {e}")
|
|
return None
|
|
|
|
def _backfill_missing_ids(self, artists: List[WatchlistArtist], provider: str):
|
|
"""
|
|
Proactively match ALL artists missing IDs for the current provider.
|
|
|
|
Example: User has 50 artists with only Spotify IDs.
|
|
When iTunes becomes active, this matches ALL 50 to iTunes in one batch.
|
|
"""
|
|
# Find artists missing IDs for the active provider (regardless of which other IDs they have)
|
|
id_attr = {
|
|
'spotify': 'spotify_artist_id',
|
|
'itunes': 'itunes_artist_id',
|
|
'deezer': 'deezer_artist_id',
|
|
}.get(provider)
|
|
|
|
if not id_attr:
|
|
logger.debug(f"Backfill not supported for provider: {provider}")
|
|
return
|
|
|
|
artists_to_match = [a for a in artists if not getattr(a, id_attr, None)]
|
|
|
|
if not artists_to_match:
|
|
logger.info(f"✅ All artists already have {provider} IDs")
|
|
return
|
|
|
|
logger.info(f"🔄 Backfilling {len(artists_to_match)} artists with {provider} IDs...")
|
|
|
|
match_fn = {
|
|
'spotify': self._match_to_spotify,
|
|
'itunes': self._match_to_itunes,
|
|
'deezer': self._match_to_deezer,
|
|
}.get(provider)
|
|
|
|
update_fn = {
|
|
'spotify': self.database.update_watchlist_spotify_id,
|
|
'itunes': self.database.update_watchlist_itunes_id,
|
|
'deezer': getattr(self.database, 'update_watchlist_deezer_id', None),
|
|
}.get(provider)
|
|
|
|
if not match_fn or not update_fn:
|
|
logger.debug(f"No match/update function available for provider: {provider}")
|
|
return
|
|
|
|
matched_count = 0
|
|
unmatched_names = []
|
|
for artist in artists_to_match:
|
|
try:
|
|
new_id = match_fn(artist.artist_name)
|
|
if new_id:
|
|
update_fn(artist.id, new_id)
|
|
setattr(artist, id_attr, new_id)
|
|
matched_count += 1
|
|
logger.info(f"✅ Matched '{artist.artist_name}' to {provider}: {new_id}")
|
|
else:
|
|
unmatched_names.append(artist.artist_name)
|
|
|
|
time.sleep(0.3)
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Could not match '{artist.artist_name}' to {provider}: {e}")
|
|
unmatched_names.append(artist.artist_name)
|
|
continue
|
|
|
|
logger.info(f"✅ Backfilled {matched_count}/{len(artists_to_match)} artists with {provider} IDs")
|
|
if unmatched_names:
|
|
logger.warning(f"⚠️ Could not confidently match {len(unmatched_names)} artists: {', '.join(unmatched_names[:10])}"
|
|
f"{'...' if len(unmatched_names) > 10 else ''} — use Watchlist Settings to link manually")
|
|
|
|
@staticmethod
|
|
def _normalize_artist_name(name: str) -> str:
|
|
"""Normalize artist name for comparison."""
|
|
if not name:
|
|
return ""
|
|
s = name.lower().strip()
|
|
# Remove "the " prefix
|
|
s = re.sub(r'^the\s+', '', s)
|
|
# Remove non-alphanumeric except spaces
|
|
s = re.sub(r'[^\w\s]', '', s)
|
|
# Collapse whitespace
|
|
s = re.sub(r'\s+', ' ', s).strip()
|
|
return s
|
|
|
|
@staticmethod
|
|
def _artist_name_similarity(name_a: str, name_b: str) -> float:
|
|
"""Calculate similarity between two artist names (0.0-1.0)."""
|
|
from difflib import SequenceMatcher
|
|
na = WatchlistScanner._normalize_artist_name(name_a)
|
|
nb = WatchlistScanner._normalize_artist_name(name_b)
|
|
if not na or not nb:
|
|
return 0.0
|
|
if na == nb:
|
|
return 1.0
|
|
return SequenceMatcher(None, na, nb).ratio()
|
|
|
|
def _best_artist_match(self, results, artist_name: str) -> Optional[str]:
|
|
"""Pick the best matching artist from search results using name similarity.
|
|
|
|
Returns the artist ID only if we're confident it's the right match.
|
|
"""
|
|
if not results:
|
|
return None
|
|
|
|
# Exact normalized match gets immediate acceptance
|
|
for r in results:
|
|
if self._normalize_artist_name(r.name) == self._normalize_artist_name(artist_name):
|
|
logger.info(f" Exact match: '{r.name}' (id={r.id})")
|
|
return r.id
|
|
|
|
# Score all results by name similarity + popularity bonus
|
|
candidates = []
|
|
for r in results:
|
|
sim = self._artist_name_similarity(artist_name, r.name)
|
|
# Small popularity bonus (max 0.05) to break ties between similar names
|
|
pop_bonus = (getattr(r, 'popularity', 0) / 100) * 0.05
|
|
score = sim + pop_bonus
|
|
candidates.append((r, sim, score))
|
|
logger.debug(f" Candidate: '{r.name}' sim={sim:.2f} pop={getattr(r, 'popularity', 0)} score={score:.3f}")
|
|
|
|
# Sort by score descending
|
|
candidates.sort(key=lambda x: x[2], reverse=True)
|
|
best, best_sim, best_score = candidates[0]
|
|
|
|
# Require high similarity to accept (0.85 threshold)
|
|
if best_sim >= 0.85:
|
|
logger.info(f" Best match: '{best.name}' (sim={best_sim:.2f}, id={best.id})")
|
|
return best.id
|
|
|
|
# Between 0.70-0.85: accept only if it's clearly better than runner-up
|
|
if best_sim >= 0.70 and len(candidates) > 1:
|
|
runner_up_sim = candidates[1][1]
|
|
if best_sim - runner_up_sim >= 0.15:
|
|
logger.info(f" Best match (clear winner): '{best.name}' (sim={best_sim:.2f}, id={best.id})")
|
|
return best.id
|
|
|
|
logger.warning(f" No confident match for '{artist_name}' — best was '{best.name}' (sim={best_sim:.2f})")
|
|
return None
|
|
|
|
def _match_to_spotify(self, artist_name: str) -> Optional[str]:
|
|
"""Match artist name to Spotify ID using fuzzy name comparison."""
|
|
try:
|
|
if hasattr(self, '_metadata_service') and self._metadata_service:
|
|
results = self._metadata_service.spotify.search_artists(artist_name, limit=5)
|
|
else:
|
|
results = self.spotify_client.search_artists(artist_name, limit=5)
|
|
|
|
return self._best_artist_match(results, artist_name)
|
|
except Exception as e:
|
|
logger.warning(f"Could not match {artist_name} to Spotify: {e}")
|
|
return None
|
|
|
|
def _match_to_itunes(self, artist_name: str) -> Optional[str]:
|
|
"""Match artist name to iTunes ID using fuzzy name comparison."""
|
|
try:
|
|
if hasattr(self, '_metadata_service') and self._metadata_service:
|
|
results = self._metadata_service.itunes.search_artists(artist_name, limit=5)
|
|
else:
|
|
logger.warning(f"Cannot match to iTunes - MetadataService not available")
|
|
return None
|
|
|
|
return self._best_artist_match(results, artist_name)
|
|
except Exception as e:
|
|
logger.warning(f"Could not match {artist_name} to iTunes: {e}")
|
|
return None
|
|
|
|
def _match_to_deezer(self, artist_name: str) -> Optional[str]:
|
|
"""Match artist name to Deezer ID using fuzzy name comparison."""
|
|
try:
|
|
# Try MetadataService fallback client (if it's Deezer)
|
|
if hasattr(self, '_metadata_service') and self._metadata_service:
|
|
client = self._metadata_service.itunes # Named 'itunes' but may be DeezerClient
|
|
from core.deezer_client import DeezerClient
|
|
if isinstance(client, DeezerClient):
|
|
results = client.search_artists(artist_name, limit=5)
|
|
return self._best_artist_match(results, artist_name)
|
|
|
|
# Fallback: create a fresh Deezer client
|
|
from core.deezer_client import DeezerClient
|
|
client = DeezerClient()
|
|
results = client.search_artists(artist_name, limit=5)
|
|
return self._best_artist_match(results, artist_name)
|
|
except Exception as e:
|
|
logger.warning(f"Could not match {artist_name} to Deezer: {e}")
|
|
return None
|
|
|
|
def _get_lookback_period_setting(self) -> str:
|
|
"""
|
|
Get the discovery lookback period setting from database.
|
|
|
|
Returns:
|
|
str: Period value ('7', '30', '90', '180', or 'all')
|
|
"""
|
|
try:
|
|
with self.database._get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT value FROM metadata WHERE key = 'discovery_lookback_period'")
|
|
row = cursor.fetchone()
|
|
|
|
if row:
|
|
return row['value']
|
|
else:
|
|
# Default to 30 days if not set
|
|
return '30'
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error getting lookback period setting, defaulting to 30 days: {e}")
|
|
return '30'
|
|
|
|
def _get_rescan_cutoff(self):
|
|
"""
|
|
Check if a lookback period change requires a one-time wider scan window.
|
|
|
|
When the lookback period is expanded, a 'watchlist_rescan_cutoff' metadata key
|
|
is set with the new cutoff date. This method returns that cutoff so the scanner
|
|
can use the wider window for artists scanned before the change. After a full
|
|
scan cycle, the key is cleared by _clear_rescan_cutoff().
|
|
|
|
Returns:
|
|
datetime cutoff if a rescan is pending with a specific date,
|
|
'all' string if lookback was set to entire discography,
|
|
None if no rescan is pending
|
|
"""
|
|
try:
|
|
with self.database._get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT value FROM metadata WHERE key = 'watchlist_rescan_cutoff'")
|
|
row = cursor.fetchone()
|
|
if row is not None:
|
|
val = row['value']
|
|
if val == '':
|
|
return 'all' # Lookback set to 'all' — scan everything
|
|
return datetime.fromisoformat(val)
|
|
except Exception as e:
|
|
logger.debug(f"Error reading rescan cutoff: {e}")
|
|
return None
|
|
|
|
def _clear_rescan_cutoff(self):
|
|
"""Clear the one-time rescan cutoff after a full scan cycle completes."""
|
|
try:
|
|
with self.database._get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("DELETE FROM metadata WHERE key = 'watchlist_rescan_cutoff'")
|
|
conn.commit()
|
|
logger.info("Cleared watchlist rescan cutoff flag")
|
|
except Exception as e:
|
|
logger.debug(f"Error clearing rescan cutoff: {e}")
|
|
|
|
def is_album_after_timestamp(self, album, timestamp: datetime) -> bool:
|
|
"""Check if album was released after the given timestamp"""
|
|
try:
|
|
if not album.release_date:
|
|
return True # Include albums with unknown release dates to be safe
|
|
|
|
# Parse release date - Spotify provides different precisions
|
|
release_date_str = album.release_date
|
|
|
|
# Handle different date formats
|
|
if len(release_date_str) == 4: # Year only (e.g., "2023")
|
|
album_date = datetime(int(release_date_str), 1, 1, tzinfo=timezone.utc)
|
|
elif len(release_date_str) == 7: # Year-month (e.g., "2023-10")
|
|
year, month = release_date_str.split('-')
|
|
album_date = datetime(int(year), int(month), 1, tzinfo=timezone.utc)
|
|
elif len(release_date_str) == 10: # Full date (e.g., "2023-10-15")
|
|
album_date = datetime.strptime(release_date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
|
elif 'T' in release_date_str: # ISO 8601 with time (e.g., "2017-12-08T08:00:00Z" from iTunes)
|
|
# Strip the time portion and parse just the date
|
|
date_part = release_date_str.split('T')[0]
|
|
album_date = datetime.strptime(date_part, "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
|
else:
|
|
logger.warning(f"Unknown release date format: {release_date_str}")
|
|
return True # Include if we can't parse
|
|
|
|
# Ensure timestamp has timezone info
|
|
if timestamp.tzinfo is None:
|
|
timestamp = timestamp.replace(tzinfo=timezone.utc)
|
|
|
|
return album_date > timestamp
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error comparing album date {album.release_date} with timestamp {timestamp}: {e}")
|
|
return True # Include if we can't determine
|
|
|
|
def _should_include_release(self, track_count: int, watchlist_artist: WatchlistArtist) -> bool:
|
|
"""
|
|
Check if a release should be included based on user's preferences.
|
|
|
|
Categorization:
|
|
- Singles: 1-3 tracks
|
|
- EPs: 4-6 tracks
|
|
- Albums: 7+ tracks
|
|
|
|
Args:
|
|
track_count: Number of tracks in the release
|
|
watchlist_artist: WatchlistArtist object with user preferences
|
|
|
|
Returns:
|
|
True if release should be included, False if should be skipped
|
|
"""
|
|
try:
|
|
# Default to including everything if preferences aren't set (backwards compatibility)
|
|
include_albums = getattr(watchlist_artist, 'include_albums', True)
|
|
include_eps = getattr(watchlist_artist, 'include_eps', True)
|
|
include_singles = getattr(watchlist_artist, 'include_singles', True)
|
|
|
|
# Determine release type based on track count
|
|
if track_count >= 7:
|
|
# This is an album
|
|
return include_albums
|
|
elif track_count >= 4:
|
|
# This is an EP (4-6 tracks)
|
|
return include_eps
|
|
else:
|
|
# This is a single (1-3 tracks)
|
|
return include_singles
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error checking release inclusion: {e}")
|
|
return True # Default to including on error
|
|
|
|
def _should_include_track(self, track, album_data, watchlist_artist: WatchlistArtist) -> bool:
|
|
"""
|
|
Check if a track should be included based on content type filters.
|
|
|
|
Filters:
|
|
- Live versions
|
|
- Remixes
|
|
- Acoustic versions
|
|
- Compilation albums
|
|
|
|
Args:
|
|
track: Track object or dict
|
|
album_data: Album data object or dict
|
|
watchlist_artist: WatchlistArtist object with user preferences
|
|
|
|
Returns:
|
|
True if track should be included, False if should be skipped
|
|
"""
|
|
try:
|
|
# Get track name and album name
|
|
if isinstance(track, dict):
|
|
track_name = track.get('name', '')
|
|
else:
|
|
track_name = getattr(track, 'name', '')
|
|
|
|
if isinstance(album_data, dict):
|
|
album_name = album_data.get('name', '')
|
|
else:
|
|
album_name = getattr(album_data, 'name', '')
|
|
|
|
# Get user preferences (default to False = exclude by default)
|
|
include_live = getattr(watchlist_artist, 'include_live', False)
|
|
include_remixes = getattr(watchlist_artist, 'include_remixes', False)
|
|
include_acoustic = getattr(watchlist_artist, 'include_acoustic', False)
|
|
include_compilations = getattr(watchlist_artist, 'include_compilations', False)
|
|
include_instrumentals = getattr(watchlist_artist, 'include_instrumentals', False)
|
|
|
|
# Check compilation albums (album-level filter)
|
|
if not include_compilations:
|
|
if is_compilation_album(album_name):
|
|
logger.debug(f"Skipping compilation album: {album_name}")
|
|
return False
|
|
|
|
# Check track content type filters
|
|
if not include_live:
|
|
if is_live_version(track_name, album_name):
|
|
logger.debug(f"Skipping live version: {track_name}")
|
|
return False
|
|
|
|
if not include_remixes:
|
|
if is_remix_version(track_name, album_name):
|
|
logger.debug(f"Skipping remix: {track_name}")
|
|
return False
|
|
|
|
if not include_acoustic:
|
|
if is_acoustic_version(track_name, album_name):
|
|
logger.debug(f"Skipping acoustic version: {track_name}")
|
|
return False
|
|
|
|
# Check instrumental versions
|
|
if not include_instrumentals:
|
|
if is_instrumental_version(track_name, album_name):
|
|
logger.debug(f"Skipping instrumental version: {track_name}")
|
|
return False
|
|
|
|
# Check custom exclusion terms
|
|
try:
|
|
from config.settings import config_manager as _cfg
|
|
exclude_terms_str = _cfg.get('watchlist.exclude_terms', '')
|
|
if exclude_terms_str:
|
|
exclude_terms = [t.strip() for t in exclude_terms_str.split(',') if t.strip()]
|
|
matched_term = matches_custom_exclude_terms(track_name, album_name, exclude_terms)
|
|
if matched_term:
|
|
logger.debug(f"Skipping track '{track_name}' — matched custom exclusion term: '{matched_term}'")
|
|
return False
|
|
except Exception as e:
|
|
logger.warning(f"Error checking custom exclusion terms: {e}")
|
|
|
|
# Track passes all filters
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error checking track content type inclusion: {e}")
|
|
return True # Default to including on error
|
|
|
|
def is_track_missing_from_library(self, track, album_name: str = None) -> bool:
|
|
"""
|
|
Check if a track is missing from the local library.
|
|
Uses the same matching logic as the download missing tracks modals.
|
|
"""
|
|
try:
|
|
# Handle both dict and object track formats
|
|
if isinstance(track, dict):
|
|
original_title = track.get('name', 'Unknown')
|
|
track_artists = track.get('artists', [])
|
|
artists_to_search = [artist.get('name', 'Unknown') for artist in track_artists] if track_artists else ["Unknown"]
|
|
else:
|
|
original_title = track.name
|
|
artists_to_search = [artist.name for artist in track.artists] if track.artists else ["Unknown"]
|
|
|
|
# Generate title variations (same logic as sync page)
|
|
title_variations = [original_title]
|
|
|
|
# Only add cleaned version if it removes clear noise
|
|
cleaned_for_search = clean_track_name_for_search(original_title)
|
|
if cleaned_for_search.lower() != original_title.lower():
|
|
title_variations.append(cleaned_for_search)
|
|
|
|
# Use matching engine's conservative clean_title
|
|
base_title = self.matching_engine.clean_title(original_title)
|
|
if base_title.lower() not in [t.lower() for t in title_variations]:
|
|
title_variations.append(base_title)
|
|
|
|
unique_title_variations = list(dict.fromkeys(title_variations))
|
|
|
|
# Search for each artist with each title variation
|
|
|
|
for artist_name in artists_to_search:
|
|
for query_title in unique_title_variations:
|
|
# Use same database check as modals with server awareness
|
|
from config.settings import config_manager
|
|
active_server = config_manager.get_active_media_server()
|
|
db_track, confidence = self.database.check_track_exists(query_title, artist_name, confidence_threshold=0.7, server_source=active_server, album=album_name)
|
|
|
|
if db_track and confidence >= 0.7:
|
|
logger.debug(f"✔️ Track found in library: '{original_title}' by '{artist_name}' (confidence: {confidence:.2f})")
|
|
return False # Track exists in library
|
|
|
|
# No match found with any variation or artist
|
|
logger.info(f"❌ Track missing from library: '{original_title}' by '{artists_to_search[0] if artists_to_search else 'Unknown'}' - adding to wishlist")
|
|
return True # Track is missing
|
|
|
|
except Exception as e:
|
|
# Handle both dict and object track formats for error logging
|
|
track_name = track.get('name', 'Unknown') if isinstance(track, dict) else getattr(track, 'name', 'Unknown')
|
|
logger.warning(f"Error checking if track exists: {track_name}: {e}")
|
|
return True # Assume missing if we can't check
|
|
|
|
def add_track_to_wishlist(self, track, album, watchlist_artist: WatchlistArtist) -> bool:
|
|
"""Add a missing track to the wishlist"""
|
|
try:
|
|
# Handle both dict and object track/album formats
|
|
if isinstance(track, dict):
|
|
track_id = track.get('id', '')
|
|
track_name = track.get('name', 'Unknown')
|
|
track_artists = track.get('artists', [])
|
|
track_duration = track.get('duration_ms', 0)
|
|
track_explicit = track.get('explicit', False)
|
|
track_external_urls = track.get('external_urls', {})
|
|
track_popularity = track.get('popularity', 0)
|
|
track_preview_url = track.get('preview_url', None)
|
|
track_number = track.get('track_number', 1)
|
|
disc_number = track.get('disc_number', 1)
|
|
track_uri = track.get('uri', '')
|
|
else:
|
|
track_id = track.id
|
|
track_name = track.name
|
|
track_artists = [{'name': artist.name, 'id': artist.id} for artist in track.artists]
|
|
track_duration = getattr(track, 'duration_ms', 0)
|
|
track_explicit = getattr(track, 'explicit', False)
|
|
track_external_urls = getattr(track, 'external_urls', {})
|
|
track_popularity = getattr(track, 'popularity', 0)
|
|
track_preview_url = getattr(track, 'preview_url', None)
|
|
track_number = getattr(track, 'track_number', 1)
|
|
disc_number = getattr(track, 'disc_number', 1)
|
|
track_uri = getattr(track, 'uri', '')
|
|
|
|
if isinstance(album, dict):
|
|
album_name = album.get('name', 'Unknown')
|
|
album_id = album.get('id', '')
|
|
album_release_date = album.get('release_date', '')
|
|
album_images = album.get('images', [])
|
|
album_type = album.get('album_type', 'album') # 'album', 'single', or 'ep'
|
|
total_tracks = album.get('total_tracks', 0)
|
|
album_artists = album.get('artists', [])
|
|
else:
|
|
album_name = album.name
|
|
album_id = album.id
|
|
album_release_date = album.release_date
|
|
album_images = album.images if hasattr(album, 'images') else []
|
|
album_type = album.album_type if hasattr(album, 'album_type') else 'album'
|
|
total_tracks = album.total_tracks if hasattr(album, 'total_tracks') else 0
|
|
album_artists = album.artists if hasattr(album, 'artists') else []
|
|
|
|
# Create Spotify track data structure
|
|
spotify_track_data = {
|
|
'id': track_id,
|
|
'name': track_name,
|
|
'artists': track_artists,
|
|
'album': {
|
|
'name': album_name,
|
|
'id': album_id,
|
|
'release_date': album_release_date,
|
|
'images': album_images,
|
|
'album_type': album_type, # Store album type for category filtering
|
|
'total_tracks': total_tracks, # Store track count for accurate categorization
|
|
'artists': album_artists
|
|
},
|
|
'duration_ms': track_duration,
|
|
'explicit': track_explicit,
|
|
'external_urls': track_external_urls,
|
|
'popularity': track_popularity,
|
|
'preview_url': track_preview_url,
|
|
'track_number': track_number,
|
|
'disc_number': disc_number,
|
|
'uri': track_uri,
|
|
'is_local': False
|
|
}
|
|
|
|
# Add to wishlist with watchlist context (scoped to artist's profile)
|
|
success = self.database.add_to_wishlist(
|
|
spotify_track_data=spotify_track_data,
|
|
failure_reason="Missing from library (found by watchlist scan)",
|
|
source_type="watchlist",
|
|
source_info={
|
|
'watchlist_artist_name': watchlist_artist.artist_name,
|
|
'watchlist_artist_id': watchlist_artist.spotify_artist_id,
|
|
'album_name': album_name,
|
|
'scan_timestamp': datetime.now().isoformat()
|
|
},
|
|
profile_id=getattr(watchlist_artist, 'profile_id', 1)
|
|
)
|
|
|
|
if success:
|
|
first_artist = track_artists[0].get('name', 'Unknown') if track_artists else 'Unknown'
|
|
logger.debug(f"Added track to wishlist: {track_name} by {first_artist}")
|
|
else:
|
|
logger.warning(f"Failed to add track to wishlist: {track_name}")
|
|
|
|
return success
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error adding track to wishlist: {track_name}: {e}")
|
|
return False
|
|
|
|
def update_artist_scan_timestamp(self, artist) -> bool:
|
|
"""Update the last scan timestamp for an artist.
|
|
|
|
Args:
|
|
artist: WatchlistArtist object, or a string spotify_artist_id for backward compat
|
|
"""
|
|
try:
|
|
with self.database._get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Support both WatchlistArtist objects and raw string IDs
|
|
if hasattr(artist, 'id'):
|
|
# WatchlistArtist object - use database primary key (always reliable)
|
|
cursor.execute("""
|
|
UPDATE watchlist_artists
|
|
SET last_scan_timestamp = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""", (artist.id,))
|
|
artist_label = f"{artist.artist_name} (id={artist.id})"
|
|
else:
|
|
# Backward compat: raw string ID (try spotify, then itunes)
|
|
cursor.execute("""
|
|
UPDATE watchlist_artists
|
|
SET last_scan_timestamp = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
|
|
WHERE spotify_artist_id = ? OR itunes_artist_id = ?
|
|
""", (artist, artist))
|
|
artist_label = f"ID {artist}"
|
|
|
|
conn.commit()
|
|
|
|
if cursor.rowcount > 0:
|
|
logger.debug(f"Updated scan timestamp for artist {artist_label}")
|
|
return True
|
|
else:
|
|
logger.warning(f"No artist found for {artist_label}")
|
|
return False
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error updating scan timestamp: {e}")
|
|
return False
|
|
|
|
def _fetch_similar_artists_from_musicmap(self, artist_name: str, limit: int = 20) -> List[Dict[str, Any]]:
|
|
"""
|
|
Fetch similar artists from MusicMap and match them to both Spotify and iTunes.
|
|
|
|
Args:
|
|
artist_name: The artist name to find similar artists for
|
|
limit: Maximum number of similar artists to return (default: 20)
|
|
|
|
Returns:
|
|
List of matched artist dictionaries with both Spotify and iTunes IDs when available
|
|
"""
|
|
try:
|
|
logger.info(f"Fetching similar artists from MusicMap for: {artist_name}")
|
|
|
|
# Construct MusicMap URL
|
|
url_artist = artist_name.lower().replace(' ', '+')
|
|
musicmap_url = f'https://www.music-map.com/{url_artist}'
|
|
|
|
# Set headers to mimic a browser
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
}
|
|
|
|
# Fetch MusicMap page
|
|
response = requests.get(musicmap_url, headers=headers, timeout=10)
|
|
response.raise_for_status()
|
|
|
|
# Parse HTML
|
|
soup = BeautifulSoup(response.text, 'html.parser')
|
|
gnod_map = soup.find(id='gnodMap')
|
|
|
|
if not gnod_map:
|
|
logger.warning(f"Could not find artist map on MusicMap for {artist_name}")
|
|
return []
|
|
|
|
# Extract similar artist names
|
|
all_anchors = gnod_map.find_all('a')
|
|
searched_artist_lower = artist_name.lower().strip()
|
|
|
|
similar_artist_names = []
|
|
for anchor in all_anchors:
|
|
artist_text = anchor.get_text(strip=True)
|
|
|
|
# Skip if this is the searched artist
|
|
if artist_text.lower() == searched_artist_lower:
|
|
continue
|
|
|
|
similar_artist_names.append(artist_text)
|
|
|
|
logger.info(f"Found {len(similar_artist_names)} similar artists from MusicMap")
|
|
|
|
# Get fallback metadata client for matching (iTunes or Deezer)
|
|
itunes_client, fallback_source = _get_fallback_metadata_client()
|
|
|
|
# Get the searched artist's IDs to exclude them
|
|
searched_spotify_id = None
|
|
searched_fallback_id = None
|
|
try:
|
|
# Try Spotify search
|
|
if self.spotify_client and self.spotify_client.is_spotify_authenticated():
|
|
searched_results = self.spotify_client.search_artists(artist_name, limit=1)
|
|
if searched_results and len(searched_results) > 0:
|
|
searched_spotify_id = searched_results[0].id
|
|
except Exception as e:
|
|
logger.debug(f"Could not get searched artist Spotify ID: {e}")
|
|
|
|
try:
|
|
# Try fallback source (iTunes/Deezer) search
|
|
fallback_results = itunes_client.search_artists(artist_name, limit=1)
|
|
if fallback_results and len(fallback_results) > 0:
|
|
searched_fallback_id = fallback_results[0].id
|
|
except Exception as e:
|
|
logger.debug(f"Could not get searched artist {fallback_source} ID: {e}")
|
|
|
|
# Match each artist to both Spotify and fallback source (iTunes/Deezer)
|
|
matched_artists = []
|
|
seen_names = set() # Track seen artist names to prevent duplicates
|
|
|
|
for artist_name_to_match in similar_artist_names[:limit]:
|
|
try:
|
|
# Skip if we've already matched this artist name
|
|
name_lower = artist_name_to_match.lower().strip()
|
|
if name_lower in seen_names:
|
|
continue
|
|
|
|
artist_data = {
|
|
'name': artist_name_to_match,
|
|
'spotify_id': None,
|
|
'itunes_id': None,
|
|
'deezer_id': None,
|
|
'image_url': None,
|
|
'genres': [],
|
|
'popularity': 0
|
|
}
|
|
|
|
# Try to match on Spotify
|
|
if self.spotify_client and self.spotify_client.is_spotify_authenticated():
|
|
try:
|
|
spotify_results = self.spotify_client.search_artists(artist_name_to_match, limit=1)
|
|
if spotify_results and len(spotify_results) > 0:
|
|
spotify_artist = spotify_results[0]
|
|
# Skip if this is the searched artist
|
|
if spotify_artist.id != searched_spotify_id:
|
|
artist_data['spotify_id'] = spotify_artist.id
|
|
artist_data['name'] = spotify_artist.name # Use canonical name
|
|
artist_data['image_url'] = spotify_artist.image_url if hasattr(spotify_artist, 'image_url') else None
|
|
artist_data['genres'] = spotify_artist.genres if hasattr(spotify_artist, 'genres') else []
|
|
artist_data['popularity'] = spotify_artist.popularity if hasattr(spotify_artist, 'popularity') else 0
|
|
except Exception as e:
|
|
logger.debug(f"Spotify match failed for {artist_name_to_match}: {e}")
|
|
|
|
# Try to match on fallback source (iTunes/Deezer) with retry for rate limiting
|
|
try:
|
|
fallback_results = itunes_api_call_with_retry(
|
|
itunes_client.search_artists, artist_name_to_match, limit=1
|
|
)
|
|
if fallback_results and len(fallback_results) > 0:
|
|
fallback_artist = fallback_results[0]
|
|
# Skip if this is the searched artist
|
|
if fallback_artist.id != searched_fallback_id:
|
|
# Store under the appropriate key based on fallback source
|
|
if fallback_source == 'deezer':
|
|
artist_data['deezer_id'] = fallback_artist.id
|
|
else:
|
|
artist_data['itunes_id'] = fallback_artist.id
|
|
# Use fallback name if we don't have Spotify
|
|
if not artist_data['spotify_id']:
|
|
artist_data['name'] = fallback_artist.name
|
|
# Use fallback genres if we don't have Spotify genres
|
|
if not artist_data['genres'] and hasattr(fallback_artist, 'genres'):
|
|
artist_data['genres'] = fallback_artist.genres
|
|
else:
|
|
logger.info(f" [{fallback_source}] No match found for: {artist_name_to_match}")
|
|
except Exception as e:
|
|
logger.info(f" [{fallback_source}] Match failed for {artist_name_to_match}: {e}")
|
|
|
|
# Only add if we got at least one ID
|
|
fallback_id_key = 'deezer_id' if fallback_source == 'deezer' else 'itunes_id'
|
|
if artist_data['spotify_id'] or artist_data.get(fallback_id_key):
|
|
seen_names.add(name_lower)
|
|
matched_artists.append(artist_data)
|
|
logger.debug(f" Matched: {artist_data['name']} (Spotify: {artist_data['spotify_id']}, {fallback_source}: {artist_data.get(fallback_id_key)})")
|
|
|
|
except Exception as match_error:
|
|
logger.debug(f"Error matching {artist_name_to_match}: {match_error}")
|
|
continue
|
|
|
|
# Log detailed matching statistics
|
|
fallback_id_key = 'deezer_id' if fallback_source == 'deezer' else 'itunes_id'
|
|
fallback_matched = sum(1 for a in matched_artists if a.get(fallback_id_key))
|
|
spotify_matched = sum(1 for a in matched_artists if a.get('spotify_id'))
|
|
both_matched = sum(1 for a in matched_artists if a.get(fallback_id_key) and a.get('spotify_id'))
|
|
logger.info(f"Matched {len(matched_artists)} similar artists - {fallback_source}: {fallback_matched}, Spotify: {spotify_matched}, Both: {both_matched}")
|
|
return matched_artists
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
logger.error(f"Error fetching from MusicMap: {e}")
|
|
return []
|
|
except Exception as e:
|
|
logger.error(f"Error fetching similar artists from MusicMap: {e}")
|
|
return []
|
|
|
|
def _backfill_similar_artists_itunes_ids(self, source_artist_id: str, profile_id: int = 1) -> int:
|
|
"""
|
|
Backfill missing iTunes IDs for cached similar artists.
|
|
This ensures seamless dual-source support without clearing cached data.
|
|
|
|
Args:
|
|
source_artist_id: The source artist ID to backfill similar artists for
|
|
profile_id: Profile to scope the backfill to
|
|
|
|
Returns:
|
|
Number of similar artists updated with iTunes IDs
|
|
"""
|
|
try:
|
|
# Get fallback metadata client (iTunes or Deezer)
|
|
fallback_client, fallback_source = _get_fallback_metadata_client()
|
|
|
|
# Get similar artists that are missing IDs for the active fallback source
|
|
similar_artists = self.database.get_similar_artists_missing_fallback_ids(source_artist_id, fallback_source, profile_id=profile_id)
|
|
|
|
if not similar_artists:
|
|
return 0
|
|
|
|
logger.info(f"Backfilling {fallback_source} IDs for {len(similar_artists)} similar artists")
|
|
|
|
updated_count = 0
|
|
for similar_artist in similar_artists:
|
|
try:
|
|
results = fallback_client.search_artists(similar_artist.similar_artist_name, limit=1)
|
|
if results and len(results) > 0:
|
|
found_id = results[0].id
|
|
# Update the similar artist with the correct source ID
|
|
if fallback_source == 'deezer':
|
|
success = self.database.update_similar_artist_deezer_id(similar_artist.id, found_id)
|
|
else:
|
|
success = self.database.update_similar_artist_itunes_id(similar_artist.id, found_id)
|
|
if success:
|
|
updated_count += 1
|
|
logger.debug(f" Backfilled {fallback_source} ID {found_id} for {similar_artist.similar_artist_name}")
|
|
except Exception as e:
|
|
logger.debug(f" Could not backfill {fallback_source} ID for {similar_artist.similar_artist_name}: {e}")
|
|
continue
|
|
|
|
if updated_count > 0:
|
|
logger.info(f"Backfilled {updated_count} similar artists with {fallback_source} IDs")
|
|
|
|
return updated_count
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error backfilling similar artists {fallback_source if 'fallback_source' in dir() else 'fallback'} IDs: {e}")
|
|
return 0
|
|
|
|
def update_similar_artists(self, watchlist_artist: WatchlistArtist, limit: int = 10, profile_id: int = 1) -> bool:
|
|
"""
|
|
Fetch and store similar artists for a watchlist artist.
|
|
Called after each artist scan to build discovery pool.
|
|
Uses MusicMap to find similar artists and matches them to both Spotify and iTunes.
|
|
"""
|
|
try:
|
|
logger.info(f"Fetching similar artists for {watchlist_artist.artist_name}")
|
|
|
|
# Get similar artists from MusicMap (returns list of artist dicts with both IDs)
|
|
similar_artists = self._fetch_similar_artists_from_musicmap(watchlist_artist.artist_name, limit=limit)
|
|
|
|
if not similar_artists:
|
|
logger.debug(f"No similar artists found for {watchlist_artist.artist_name}")
|
|
return True # Not an error, just no recommendations
|
|
|
|
logger.info(f"Found {len(similar_artists)} similar artists for {watchlist_artist.artist_name}")
|
|
|
|
# Use consistent source artist ID (prefer Spotify, fall back to iTunes or internal ID)
|
|
source_artist_id = watchlist_artist.spotify_artist_id or watchlist_artist.itunes_artist_id or str(watchlist_artist.id)
|
|
|
|
# Store each similar artist in database
|
|
stored_count = 0
|
|
for rank, similar_artist in enumerate(similar_artists, 1):
|
|
try:
|
|
# similar_artist has 'name', 'spotify_id', 'itunes_id', 'deezer_id', 'image_url', 'genres', 'popularity'
|
|
success = self.database.add_or_update_similar_artist(
|
|
source_artist_id=source_artist_id,
|
|
similar_artist_name=similar_artist['name'],
|
|
similar_artist_spotify_id=similar_artist.get('spotify_id'),
|
|
similar_artist_itunes_id=similar_artist.get('itunes_id'),
|
|
similarity_rank=rank,
|
|
profile_id=profile_id,
|
|
image_url=similar_artist.get('image_url'),
|
|
genres=similar_artist.get('genres'),
|
|
popularity=similar_artist.get('popularity', 0),
|
|
similar_artist_deezer_id=similar_artist.get('deezer_id')
|
|
)
|
|
|
|
if success:
|
|
stored_count += 1
|
|
fallback_id = similar_artist.get('deezer_id') or similar_artist.get('itunes_id')
|
|
fallback_label = 'Deezer' if similar_artist.get('deezer_id') else 'iTunes'
|
|
logger.debug(f" #{rank}: {similar_artist['name']} (Spotify: {similar_artist.get('spotify_id')}, {fallback_label}: {fallback_id})")
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error storing similar artist {similar_artist.get('name', 'Unknown')}: {e}")
|
|
continue
|
|
|
|
logger.info(f"Stored {stored_count}/{len(similar_artists)} similar artists for {watchlist_artist.artist_name}")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching similar artists for {watchlist_artist.artist_name}: {e}")
|
|
return False
|
|
|
|
def populate_discovery_pool(self, top_artists_limit: int = 50, albums_per_artist: int = 10, profile_id: int = 1, progress_callback=None):
|
|
"""
|
|
Populate discovery pool with tracks from top similar artists.
|
|
Called after watchlist scan completes.
|
|
|
|
Supports both Spotify and iTunes sources - populates for whichever is available.
|
|
- Checks if pool was updated in last 24 hours (prevents over-polling)
|
|
- Includes albums, singles, and EPs for comprehensive coverage
|
|
- Appends to existing pool instead of replacing it
|
|
- Cleans up tracks older than 365 days (maintains 1 year rolling window)
|
|
"""
|
|
try:
|
|
from datetime import datetime, timedelta
|
|
import random
|
|
|
|
# Check if we should run discovery pool population (prevents over-polling)
|
|
skip_pool_population = not self.database.should_populate_discovery_pool(hours_threshold=24, profile_id=profile_id)
|
|
|
|
if skip_pool_population:
|
|
logger.info("Discovery pool was populated recently (< 24 hours ago). Skipping pool population.")
|
|
logger.info("But still refreshing recent albums cache and curated playlists...")
|
|
if progress_callback:
|
|
progress_callback('skip', 'Discovery pool recently updated, skipping')
|
|
# Still run these even when skipping main pool population
|
|
if progress_callback:
|
|
progress_callback('phase', 'Caching recent albums...')
|
|
self.cache_discovery_recent_albums(profile_id=profile_id)
|
|
if progress_callback:
|
|
progress_callback('phase', 'Curating playlists...')
|
|
self.curate_discovery_playlists(profile_id=profile_id)
|
|
return
|
|
|
|
logger.info("Populating discovery pool from similar artists...")
|
|
|
|
# Determine which sources are available
|
|
spotify_available = self.spotify_client and self.spotify_client.is_spotify_authenticated()
|
|
|
|
# Import fallback metadata client (iTunes or Deezer)
|
|
itunes_client, fallback_source = _get_fallback_metadata_client()
|
|
fallback_available = True # Fallback source is always available (no auth needed)
|
|
|
|
if not spotify_available and not fallback_available:
|
|
logger.warning("No music sources available to populate discovery pool")
|
|
return
|
|
|
|
logger.info(f"Sources available - Spotify: {spotify_available}, {fallback_source}: {fallback_available}")
|
|
|
|
# Get top similar artists for this profile's watchlist (ordered by occurrence_count)
|
|
similar_artists = self.database.get_top_similar_artists(limit=top_artists_limit, profile_id=profile_id)
|
|
|
|
if not similar_artists:
|
|
logger.info("No similar artists found to populate discovery pool from similar artists")
|
|
logger.info("But still caching recent albums from watchlist artists and curating playlists...")
|
|
if progress_callback:
|
|
progress_callback('skip', 'No similar artists found')
|
|
# Still run these even without similar artists - they use watchlist artists
|
|
if progress_callback:
|
|
progress_callback('phase', 'Caching recent albums...')
|
|
self.cache_discovery_recent_albums(profile_id=profile_id)
|
|
if progress_callback:
|
|
progress_callback('phase', 'Curating playlists...')
|
|
self.curate_discovery_playlists(profile_id=profile_id)
|
|
return
|
|
|
|
logger.info(f"Processing {len(similar_artists)} top similar artists for discovery pool")
|
|
|
|
total_tracks_added = 0
|
|
|
|
for artist_idx, similar_artist in enumerate(similar_artists, 1):
|
|
try:
|
|
logger.info(f"[{artist_idx}/{len(similar_artists)}] Processing {similar_artist.similar_artist_name} (occurrence: {similar_artist.occurrence_count})")
|
|
if progress_callback:
|
|
progress_callback('artist', f'{similar_artist.similar_artist_name} ({artist_idx}/{len(similar_artists)})')
|
|
|
|
# Build list of sources to process for this artist
|
|
# Fallback source (iTunes/Deezer) is ALWAYS processed (baseline), Spotify is added if authenticated
|
|
sources_to_process = []
|
|
|
|
# Always add fallback source first (baseline source)
|
|
fallback_id = similar_artist.similar_artist_itunes_id if fallback_source == 'itunes' else getattr(similar_artist, 'similar_artist_deezer_id', None)
|
|
if not fallback_id:
|
|
# On-the-fly lookup for missing fallback ID (seamless provider switching)
|
|
try:
|
|
fallback_results = itunes_client.search_artists(similar_artist.similar_artist_name, limit=1)
|
|
if fallback_results and len(fallback_results) > 0:
|
|
fallback_id = fallback_results[0].id
|
|
# Cache it for future use
|
|
if fallback_source == 'deezer':
|
|
self.database.update_similar_artist_deezer_id(similar_artist.id, fallback_id)
|
|
else:
|
|
self.database.update_similar_artist_itunes_id(similar_artist.id, fallback_id)
|
|
logger.debug(f" Resolved {fallback_source} ID {fallback_id} for {similar_artist.similar_artist_name}")
|
|
except Exception as e:
|
|
logger.debug(f" Could not resolve {fallback_source} ID for {similar_artist.similar_artist_name}: {e}")
|
|
|
|
if fallback_id:
|
|
sources_to_process.append((fallback_source, fallback_id))
|
|
|
|
# Add Spotify if authenticated and we have an ID
|
|
if spotify_available and similar_artist.similar_artist_spotify_id:
|
|
sources_to_process.append(('spotify', similar_artist.similar_artist_spotify_id))
|
|
|
|
if not sources_to_process:
|
|
logger.debug(f"No valid IDs for {similar_artist.similar_artist_name}, skipping")
|
|
continue
|
|
|
|
logger.debug(f" Processing {len(sources_to_process)} source(s): {[s[0] for s in sources_to_process]}")
|
|
|
|
# Process each source for this artist
|
|
for source, artist_id in sources_to_process:
|
|
try:
|
|
# Get artist's albums from this source
|
|
if source == 'spotify':
|
|
all_albums = self.spotify_client.get_artist_albums(
|
|
artist_id,
|
|
album_type='album,single,ep',
|
|
limit=50,
|
|
skip_cache=True
|
|
)
|
|
else: # itunes or deezer fallback
|
|
all_albums = itunes_client.get_artist_albums(
|
|
artist_id,
|
|
album_type='album,single,ep',
|
|
limit=50
|
|
)
|
|
|
|
if not all_albums:
|
|
logger.debug(f"No albums found for {similar_artist.similar_artist_name} on {source}")
|
|
continue
|
|
|
|
# Fetch artist genres for this source
|
|
artist_genres = []
|
|
try:
|
|
if source == 'spotify':
|
|
artist_data = self.spotify_client.get_artist(artist_id)
|
|
if artist_data and 'genres' in artist_data:
|
|
artist_genres = artist_data['genres']
|
|
else: # itunes/deezer - genres from artist lookup
|
|
artist_data = itunes_client.get_artist(artist_id)
|
|
if artist_data and 'genres' in artist_data:
|
|
artist_genres = artist_data['genres']
|
|
except Exception as e:
|
|
logger.debug(f"Could not fetch genres for {similar_artist.similar_artist_name} on {source}: {e}")
|
|
|
|
# IMPROVED: Smart selection mixing albums, singles, and EPs
|
|
# Prioritize recent releases and popular content
|
|
|
|
# Separate by type for balanced selection
|
|
albums = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type == 'album']
|
|
singles_eps = [a for a in all_albums if hasattr(a, 'album_type') and a.album_type in ['single', 'ep']]
|
|
other = [a for a in all_albums if not hasattr(a, 'album_type')]
|
|
|
|
# Select albums: latest releases + popular older content
|
|
selected_albums = []
|
|
|
|
# Always include 3 most recent releases (any type) - this captures new singles/EPs
|
|
latest_releases = all_albums[:3]
|
|
selected_albums.extend(latest_releases)
|
|
|
|
# Add remaining slots with balanced mix
|
|
remaining_slots = albums_per_artist - len(selected_albums)
|
|
if remaining_slots > 0:
|
|
# Combine remaining albums and singles
|
|
remaining_content = all_albums[3:]
|
|
|
|
if len(remaining_content) > remaining_slots:
|
|
# Randomly select from remaining content
|
|
random_selection = random.sample(remaining_content, remaining_slots)
|
|
selected_albums.extend(random_selection)
|
|
else:
|
|
selected_albums.extend(remaining_content)
|
|
|
|
logger.info(f" [{source}] Selected {len(selected_albums)} releases from {len(all_albums)} available (albums: {len(albums)}, singles/EPs: {len(singles_eps)})")
|
|
|
|
# Process each selected album
|
|
for album_idx, album in enumerate(selected_albums, 1):
|
|
try:
|
|
# Get full album data with tracks from appropriate source
|
|
if source == 'spotify':
|
|
album_data = self.spotify_client.get_album(album.id)
|
|
if not album_data or 'tracks' not in album_data:
|
|
continue
|
|
tracks = album_data['tracks'].get('items', [])
|
|
else: # itunes or deezer fallback
|
|
album_data = itunes_client.get_album(album.id)
|
|
if not album_data:
|
|
continue
|
|
# get_album includes tracks by default (include_tracks=True)
|
|
tracks = album_data.get('tracks', {}).get('items', [])
|
|
|
|
logger.debug(f" Album {album_idx}: {album_data.get('name', 'Unknown')} ({len(tracks)} tracks)")
|
|
|
|
# Determine if this is a new release (within last 30 days)
|
|
is_new = False
|
|
try:
|
|
release_date_str = album_data.get('release_date', '')
|
|
if release_date_str:
|
|
# Handle full date or year-only
|
|
if len(release_date_str) >= 10:
|
|
release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d")
|
|
days_old = (datetime.now() - release_date).days
|
|
is_new = days_old <= 30
|
|
except:
|
|
pass
|
|
|
|
# Add each track to discovery pool
|
|
for track in tracks:
|
|
try:
|
|
# Enhance track object with full album data (including album_type)
|
|
enhanced_track = {
|
|
**track,
|
|
'album': {
|
|
'id': album_data['id'],
|
|
'name': album_data.get('name', 'Unknown Album'),
|
|
'images': album_data.get('images', []),
|
|
'release_date': album_data.get('release_date', ''),
|
|
'album_type': album_data.get('album_type', 'album'),
|
|
'total_tracks': album_data.get('total_tracks', 0)
|
|
},
|
|
'_source': source
|
|
}
|
|
|
|
# Build track data for discovery pool with source-specific IDs
|
|
# iTunes/Deezer have no popularity data — synthesize from recency + occurrence
|
|
raw_popularity = album_data.get('popularity', 0)
|
|
if source in ('itunes', 'deezer') and raw_popularity == 0:
|
|
# Base 45, boost by recency and artist occurrence count
|
|
synth_pop = 45
|
|
if is_new:
|
|
synth_pop += 25 # New releases get a big boost
|
|
else:
|
|
try:
|
|
release_str = album_data.get('release_date', '')
|
|
if release_str and len(release_str) >= 10:
|
|
rel_date = datetime.strptime(release_str[:10], "%Y-%m-%d")
|
|
age_days = (datetime.now() - rel_date).days
|
|
if age_days <= 90:
|
|
synth_pop += 15
|
|
elif age_days <= 365:
|
|
synth_pop += 5
|
|
except:
|
|
pass
|
|
# Artists that appear similar to multiple watchlist artists are likely more relevant
|
|
if similar_artist.occurrence_count >= 3:
|
|
synth_pop += 10
|
|
elif similar_artist.occurrence_count >= 2:
|
|
synth_pop += 5
|
|
raw_popularity = min(synth_pop, 100)
|
|
|
|
track_data = {
|
|
'track_name': track.get('name', 'Unknown Track'),
|
|
'artist_name': similar_artist.similar_artist_name,
|
|
'album_name': album_data.get('name', 'Unknown Album'),
|
|
'album_cover_url': album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None,
|
|
'duration_ms': track.get('duration_ms', 0),
|
|
'popularity': raw_popularity,
|
|
'release_date': album_data.get('release_date', ''),
|
|
'is_new_release': is_new,
|
|
'track_data_json': enhanced_track,
|
|
'artist_genres': artist_genres
|
|
}
|
|
|
|
# Add source-specific IDs
|
|
if source == 'spotify':
|
|
track_data['spotify_track_id'] = track.get('id')
|
|
track_data['spotify_album_id'] = album_data.get('id')
|
|
track_data['spotify_artist_id'] = similar_artist.similar_artist_spotify_id
|
|
elif source == 'deezer':
|
|
track_data['deezer_track_id'] = track.get('id')
|
|
track_data['deezer_album_id'] = album_data.get('id')
|
|
track_data['deezer_artist_id'] = getattr(similar_artist, 'similar_artist_deezer_id', None)
|
|
else: # itunes
|
|
track_data['itunes_track_id'] = track.get('id')
|
|
track_data['itunes_album_id'] = album_data.get('id')
|
|
track_data['itunes_artist_id'] = similar_artist.similar_artist_itunes_id
|
|
|
|
# Add to discovery pool with source (scoped to profile)
|
|
if self.database.add_to_discovery_pool(track_data, source=source, profile_id=profile_id):
|
|
total_tracks_added += 1
|
|
|
|
except Exception as track_error:
|
|
logger.debug(f"Error adding track to discovery pool: {track_error}")
|
|
continue
|
|
|
|
# Small delay between albums
|
|
time.sleep(DELAY_BETWEEN_ALBUMS)
|
|
|
|
except Exception as album_error:
|
|
logger.warning(f"Error processing album on {source}: {album_error}")
|
|
continue
|
|
|
|
except Exception as source_error:
|
|
logger.warning(f"Error processing {source} source for {similar_artist.similar_artist_name}: {source_error}")
|
|
continue
|
|
|
|
# Delay between artists (after processing all sources for this artist)
|
|
if artist_idx < len(similar_artists):
|
|
time.sleep(DELAY_BETWEEN_ARTISTS)
|
|
|
|
except Exception as artist_error:
|
|
logger.warning(f"Error processing artist {similar_artist.similar_artist_name}: {artist_error}")
|
|
continue
|
|
|
|
logger.info(f"Discovery pool from similar artists complete: {total_tracks_added} tracks added")
|
|
if progress_callback:
|
|
progress_callback('success', f'Discovery pool: {total_tracks_added} tracks from {len(similar_artists)} artists')
|
|
|
|
# Note: Watchlist artist albums are already in discovery pool from the watchlist scan itself
|
|
# No need to re-fetch them here to avoid duplicate API calls
|
|
|
|
# Add tracks from random database albums for extra variety (reduced to 5 to save API calls)
|
|
logger.info("Adding tracks from database albums to discovery pool...")
|
|
try:
|
|
with self.database._get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT DISTINCT a.title, ar.name as artist_name
|
|
FROM albums a
|
|
JOIN artists ar ON a.artist_id = ar.id
|
|
ORDER BY RANDOM()
|
|
LIMIT 5
|
|
""")
|
|
db_albums = cursor.fetchall()
|
|
|
|
logger.info(f"Processing {len(db_albums)} database albums for discovery pool")
|
|
|
|
for db_idx, album_row in enumerate(db_albums, 1):
|
|
try:
|
|
query = f"{album_row['title']} {album_row['artist_name']}"
|
|
album_data = None
|
|
tracks = []
|
|
db_source = None
|
|
artist_id_for_genres = None
|
|
|
|
# Try Spotify first if available
|
|
if spotify_available:
|
|
try:
|
|
search_results = self.spotify_client.search_albums(f"album:{album_row['title']} artist:{album_row['artist_name']}", limit=1)
|
|
if search_results and len(search_results) > 0:
|
|
spotify_album = search_results[0]
|
|
album_data = self.spotify_client.get_album(spotify_album.id)
|
|
if album_data and 'tracks' in album_data:
|
|
tracks = album_data['tracks'].get('items', [])
|
|
db_source = 'spotify'
|
|
if album_data.get('artists'):
|
|
artist_id_for_genres = album_data['artists'][0]['id']
|
|
except Exception as e:
|
|
logger.debug(f"Spotify search failed for {album_row['title']}: {e}")
|
|
|
|
# Fall back to fallback source (iTunes/Deezer) if Spotify didn't work
|
|
if not tracks and fallback_available:
|
|
try:
|
|
search_results = itunes_client.search_albums(query, limit=1)
|
|
if search_results and len(search_results) > 0:
|
|
fallback_album = search_results[0]
|
|
album_data = itunes_client.get_album(fallback_album.id)
|
|
if album_data:
|
|
tracks_data = itunes_client.get_album_tracks(fallback_album.id)
|
|
tracks = tracks_data.get('items', []) if tracks_data else []
|
|
db_source = fallback_source
|
|
# Artist ID is in the album data
|
|
if album_data.get('artists'):
|
|
artist_id_for_genres = album_data['artists'][0].get('id')
|
|
except Exception as e:
|
|
logger.debug(f"{fallback_source} search failed for {album_row['title']}: {e}")
|
|
|
|
if not tracks or not album_data:
|
|
continue
|
|
|
|
# Fetch artist genres
|
|
artist_genres = []
|
|
try:
|
|
if artist_id_for_genres:
|
|
if db_source == 'spotify':
|
|
artist_data = self.spotify_client.get_artist(artist_id_for_genres)
|
|
else:
|
|
artist_data = itunes_client.get_artist(artist_id_for_genres)
|
|
if artist_data and 'genres' in artist_data:
|
|
artist_genres = artist_data['genres']
|
|
except Exception as e:
|
|
logger.debug(f"Could not fetch genres for album artist: {e}")
|
|
|
|
# Check if new release
|
|
is_new = False
|
|
try:
|
|
release_date_str = album_data.get('release_date', '')
|
|
if release_date_str and len(release_date_str) >= 10:
|
|
release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d")
|
|
days_old = (datetime.now() - release_date).days
|
|
is_new = days_old <= 30
|
|
except:
|
|
pass
|
|
|
|
for track in tracks:
|
|
try:
|
|
enhanced_track = {
|
|
**track,
|
|
'album': {
|
|
'id': album_data['id'],
|
|
'name': album_row['title'],
|
|
'images': album_data.get('images', []),
|
|
'release_date': album_data.get('release_date', ''),
|
|
'album_type': album_data.get('album_type', 'album'),
|
|
'total_tracks': album_data.get('total_tracks', 0)
|
|
},
|
|
'_source': db_source
|
|
}
|
|
|
|
track_data = {
|
|
'track_name': track.get('name', 'Unknown Track'),
|
|
'artist_name': album_row['artist_name'],
|
|
'album_name': album_row['title'],
|
|
'album_cover_url': album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None,
|
|
'duration_ms': track.get('duration_ms', 0),
|
|
'popularity': album_data.get('popularity', 0),
|
|
'release_date': album_data.get('release_date', ''),
|
|
'is_new_release': is_new,
|
|
'track_data_json': enhanced_track,
|
|
'artist_genres': artist_genres
|
|
}
|
|
|
|
# Add source-specific IDs
|
|
if db_source == 'spotify':
|
|
track_data['spotify_track_id'] = track.get('id')
|
|
track_data['spotify_album_id'] = album_data.get('id')
|
|
track_data['spotify_artist_id'] = artist_id_for_genres or ''
|
|
elif db_source == 'deezer':
|
|
track_data['deezer_track_id'] = track.get('id')
|
|
track_data['deezer_album_id'] = album_data.get('id')
|
|
track_data['deezer_artist_id'] = artist_id_for_genres or ''
|
|
else: # itunes
|
|
track_data['itunes_track_id'] = track.get('id')
|
|
track_data['itunes_album_id'] = album_data.get('id')
|
|
track_data['itunes_artist_id'] = artist_id_for_genres or ''
|
|
|
|
if self.database.add_to_discovery_pool(track_data, source=db_source, profile_id=profile_id):
|
|
total_tracks_added += 1
|
|
except Exception as track_error:
|
|
continue
|
|
|
|
time.sleep(DELAY_BETWEEN_ALBUMS)
|
|
except Exception as album_error:
|
|
logger.debug(f"Error processing database album {album_row['title']}: {album_error}")
|
|
continue
|
|
|
|
# Rate limit between albums
|
|
if db_idx < len(db_albums):
|
|
time.sleep(DELAY_BETWEEN_ARTISTS)
|
|
|
|
except Exception as db_error:
|
|
logger.warning(f"Error processing database albums: {db_error}")
|
|
|
|
logger.info(f"Discovery pool population complete: {total_tracks_added} total tracks added from all sources")
|
|
|
|
# Clean up tracks older than 365 days (maintain 1 year rolling window)
|
|
logger.info("Cleaning up discovery tracks older than 365 days...")
|
|
deleted_count = self.database.cleanup_old_discovery_tracks(days_threshold=365)
|
|
logger.info(f"Cleaned up {deleted_count} old tracks from discovery pool")
|
|
|
|
# Get final track count for metadata
|
|
with self.database._get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT COUNT(*) as count FROM discovery_pool")
|
|
final_count = cursor.fetchone()['count']
|
|
|
|
# Update timestamp to mark when pool was last populated
|
|
self.database.update_discovery_pool_timestamp(track_count=final_count, profile_id=profile_id)
|
|
logger.info(f"Discovery pool now contains {final_count} total tracks (built over time)")
|
|
|
|
# Cache recent albums for discovery page
|
|
logger.info("Caching recent albums for discovery page...")
|
|
if progress_callback:
|
|
progress_callback('phase', 'Caching recent albums...')
|
|
self.cache_discovery_recent_albums(profile_id=profile_id)
|
|
|
|
# Curate playlists for consistent daily experience
|
|
logger.info("Curating discovery playlists...")
|
|
if progress_callback:
|
|
progress_callback('phase', 'Curating playlists...')
|
|
self.curate_discovery_playlists(profile_id=profile_id)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error populating discovery pool: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
def update_discovery_pool_incremental(self, profile_id: int = 1):
|
|
"""
|
|
Lightweight incremental update for discovery pool - runs every 6 hours.
|
|
|
|
IMPROVED: Quick check for new releases from watchlist artists only
|
|
- Much faster than full populate_discovery_pool (only checks watchlist, not similar artists)
|
|
- Only fetches latest 5 releases per artist
|
|
- Only adds tracks from releases in last 7 days
|
|
- Respects 6-hour cooldown to avoid over-polling
|
|
"""
|
|
try:
|
|
from datetime import datetime, timedelta
|
|
|
|
# Check if we should run (prevents over-polling Spotify)
|
|
if not self.database.should_populate_discovery_pool(hours_threshold=6, profile_id=profile_id):
|
|
logger.info("Discovery pool was updated recently (< 6 hours ago). Skipping incremental update.")
|
|
return
|
|
|
|
logger.info("Starting incremental discovery pool update (watchlist artists only)...")
|
|
|
|
watchlist_artists = self.database.get_watchlist_artists(profile_id=profile_id)
|
|
if not watchlist_artists:
|
|
logger.info("No watchlist artists to check for incremental update")
|
|
return
|
|
|
|
cutoff_date = datetime.now() - timedelta(days=7) # Only last week's releases
|
|
total_tracks_added = 0
|
|
|
|
for artist_idx, artist in enumerate(watchlist_artists, 1):
|
|
try:
|
|
logger.info(f"[{artist_idx}/{len(watchlist_artists)}] Checking {artist.artist_name} for new releases...")
|
|
|
|
# Only fetch latest 5 releases (much faster than full scan)
|
|
recent_releases = self.spotify_client.get_artist_albums(
|
|
artist.spotify_artist_id,
|
|
album_type='album,single,ep',
|
|
limit=5,
|
|
skip_cache=True
|
|
)
|
|
|
|
if not recent_releases:
|
|
continue
|
|
|
|
# Fetch artist genres once for all tracks of this artist
|
|
artist_genres = []
|
|
try:
|
|
artist_data = self.spotify_client.get_artist(artist.spotify_artist_id)
|
|
if artist_data and 'genres' in artist_data:
|
|
artist_genres = artist_data['genres']
|
|
except Exception as e:
|
|
logger.debug(f"Could not fetch genres for {artist.artist_name}: {e}")
|
|
|
|
for release in recent_releases:
|
|
try:
|
|
# Check if release is within cutoff
|
|
if not self.is_album_after_timestamp(release, cutoff_date):
|
|
continue # Skip older releases
|
|
|
|
# Get full album data with tracks
|
|
album_data = self.spotify_client.get_album(release.id)
|
|
if not album_data or 'tracks' not in album_data:
|
|
continue
|
|
|
|
tracks = album_data['tracks'].get('items', [])
|
|
logger.debug(f" New release: {release.name} ({len(tracks)} tracks)")
|
|
|
|
# Determine if this is a new release (within last 30 days)
|
|
is_new = False
|
|
try:
|
|
release_date_str = album_data.get('release_date', '')
|
|
if release_date_str and len(release_date_str) == 10:
|
|
release_date = datetime.strptime(release_date_str, "%Y-%m-%d")
|
|
days_old = (datetime.now() - release_date).days
|
|
is_new = days_old <= 30
|
|
except:
|
|
pass
|
|
|
|
# Add each track to discovery pool
|
|
for track in tracks:
|
|
try:
|
|
# Enhance track object with full album data (including album_type)
|
|
enhanced_track = {
|
|
**track,
|
|
'album': {
|
|
'id': album_data['id'],
|
|
'name': album_data.get('name', 'Unknown Album'),
|
|
'images': album_data.get('images', []),
|
|
'release_date': album_data.get('release_date', ''),
|
|
'album_type': album_data.get('album_type', 'album'),
|
|
'total_tracks': album_data.get('total_tracks', 0)
|
|
}
|
|
}
|
|
|
|
track_data = {
|
|
'spotify_track_id': track['id'],
|
|
'spotify_album_id': album_data['id'],
|
|
'spotify_artist_id': artist.spotify_artist_id,
|
|
'track_name': track['name'],
|
|
'artist_name': artist.artist_name,
|
|
'album_name': album_data.get('name', 'Unknown Album'),
|
|
'album_cover_url': album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None,
|
|
'duration_ms': track.get('duration_ms', 0),
|
|
'popularity': album_data.get('popularity', 0),
|
|
'release_date': album_data.get('release_date', ''),
|
|
'is_new_release': is_new,
|
|
'track_data_json': enhanced_track, # Store enhanced track with full album data
|
|
'artist_genres': artist_genres
|
|
}
|
|
|
|
if self.database.add_to_discovery_pool(track_data, profile_id=profile_id):
|
|
total_tracks_added += 1
|
|
|
|
except Exception as track_error:
|
|
logger.debug(f"Error adding track to discovery pool: {track_error}")
|
|
continue
|
|
|
|
except Exception as release_error:
|
|
logger.warning(f"Error processing release: {release_error}")
|
|
continue
|
|
|
|
# Small delay between artists
|
|
if artist_idx < len(watchlist_artists):
|
|
time.sleep(DELAY_BETWEEN_ARTISTS)
|
|
|
|
except Exception as artist_error:
|
|
logger.warning(f"Error checking {artist.artist_name}: {artist_error}")
|
|
continue
|
|
|
|
logger.info(f"Incremental update complete: {total_tracks_added} new tracks added from watchlist artists")
|
|
|
|
# Update timestamp
|
|
if total_tracks_added > 0:
|
|
# Get current track count
|
|
with self.database._get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT COUNT(*) as count FROM discovery_pool")
|
|
current_count = cursor.fetchone()['count']
|
|
|
|
self.database.update_discovery_pool_timestamp(track_count=current_count, profile_id=profile_id)
|
|
logger.info(f"Discovery pool now contains {current_count} total tracks")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error during incremental discovery pool update: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
def cache_discovery_recent_albums(self, profile_id: int = 1):
|
|
"""
|
|
Cache recent albums from watchlist and similar artists for discover page.
|
|
|
|
Supports both Spotify and iTunes sources - iTunes is always processed (baseline),
|
|
Spotify is added when authenticated. Same pattern as discovery pool.
|
|
"""
|
|
try:
|
|
from datetime import datetime, timedelta
|
|
|
|
logger.info("Caching recent albums for discover page...")
|
|
|
|
# Clear existing cache for this profile
|
|
self.database.clear_discovery_recent_albums(profile_id=profile_id)
|
|
|
|
# Adaptive window based on listening velocity
|
|
days_lookback = 30
|
|
try:
|
|
profile = self._get_listening_profile(profile_id)
|
|
if profile['has_data']:
|
|
if profile['avg_daily_plays'] < 5:
|
|
days_lookback = 60 # Casual listener — show more
|
|
elif profile['avg_daily_plays'] > 20:
|
|
days_lookback = 21 # Heavy listener — keep it fresh
|
|
logger.info(f"Recent albums window: {days_lookback} days (avg {profile['avg_daily_plays']:.1f} plays/day)")
|
|
except Exception:
|
|
pass
|
|
cutoff_date = datetime.now() - timedelta(days=days_lookback)
|
|
cached_count = {'spotify': 0, 'itunes': 0, 'deezer': 0}
|
|
albums_checked = 0
|
|
|
|
# Determine available sources
|
|
spotify_available = self.spotify_client and self.spotify_client.is_spotify_authenticated()
|
|
|
|
# Get fallback metadata client (iTunes or Deezer)
|
|
itunes_client, fallback_source = _get_fallback_metadata_client()
|
|
|
|
# Get artists to check (scoped to profile)
|
|
watchlist_artists = self.database.get_watchlist_artists(profile_id=profile_id)
|
|
similar_artists = self.database.get_top_similar_artists(limit=50, profile_id=profile_id)
|
|
|
|
logger.info(f"Checking albums from {len(watchlist_artists)} watchlist + {len(similar_artists)} similar artists")
|
|
logger.info(f"Sources: Spotify={spotify_available}, {fallback_source}=True")
|
|
|
|
def process_album(album, artist_name, artist_spotify_id, artist_itunes_id, source, artist_deezer_id=None):
|
|
"""Helper to process and cache a single album"""
|
|
nonlocal albums_checked
|
|
try:
|
|
albums_checked += 1
|
|
release_str = album.release_date if hasattr(album, 'release_date') else None
|
|
|
|
if not release_str:
|
|
return False
|
|
|
|
# Handle iTunes/Deezer ISO format (2017-12-08T08:00:00Z)
|
|
if 'T' in release_str:
|
|
release_str = release_str.split('T')[0]
|
|
|
|
if len(release_str) >= 10:
|
|
release_date = datetime.strptime(release_str[:10], "%Y-%m-%d")
|
|
if release_date >= cutoff_date:
|
|
album_data = {
|
|
'album_spotify_id': album.id if source == 'spotify' else None,
|
|
'album_itunes_id': album.id if source == 'itunes' else None,
|
|
'album_deezer_id': album.id if source == 'deezer' else None,
|
|
'album_name': album.name,
|
|
'artist_name': artist_name,
|
|
'artist_spotify_id': artist_spotify_id,
|
|
'artist_itunes_id': artist_itunes_id,
|
|
'artist_deezer_id': artist_deezer_id,
|
|
'album_cover_url': album.image_url if hasattr(album, 'image_url') else None,
|
|
'release_date': release_str[:10],
|
|
'album_type': album.album_type if hasattr(album, 'album_type') else 'album'
|
|
}
|
|
if self.database.cache_discovery_recent_album(album_data, source=source, profile_id=profile_id):
|
|
cached_count[source] += 1
|
|
logger.debug(f"Cached [{source}] recent album: {album.name} by {artist_name} ({release_str})")
|
|
return True
|
|
except Exception as e:
|
|
logger.debug(f"Error processing album: {e}")
|
|
return False
|
|
|
|
# Track resolution stats
|
|
fallback_resolved = 0
|
|
fallback_failed_resolve = 0
|
|
|
|
# Process watchlist artists
|
|
for artist in watchlist_artists:
|
|
# Always process fallback source (iTunes or Deezer) as baseline
|
|
fallback_id = artist.itunes_artist_id if fallback_source == 'itunes' else artist.deezer_artist_id
|
|
if not fallback_id:
|
|
# Try to resolve fallback ID on-the-fly (with retry for rate limiting)
|
|
try:
|
|
results = itunes_api_call_with_retry(
|
|
itunes_client.search_artists, artist.artist_name, limit=1
|
|
)
|
|
if results and len(results) > 0:
|
|
fallback_id = results[0].id
|
|
fallback_resolved += 1
|
|
logger.debug(f"[{fallback_source}] Resolved ID for {artist.artist_name}: {fallback_id}")
|
|
else:
|
|
fallback_failed_resolve += 1
|
|
logger.info(f"[{fallback_source}] No artist found for: {artist.artist_name}")
|
|
except Exception as e:
|
|
fallback_failed_resolve += 1
|
|
logger.info(f"[{fallback_source}] Failed to resolve {artist.artist_name}: {e}")
|
|
|
|
if fallback_id:
|
|
try:
|
|
albums = itunes_api_call_with_retry(
|
|
itunes_client.get_artist_albums, fallback_id, album_type='album,single,ep', limit=20
|
|
)
|
|
for album in albums or []:
|
|
process_album(
|
|
album, artist.artist_name, artist.spotify_artist_id,
|
|
fallback_id if fallback_source == 'itunes' else None,
|
|
fallback_source,
|
|
artist_deezer_id=fallback_id if fallback_source == 'deezer' else None
|
|
)
|
|
except Exception as e:
|
|
logger.info(f"[{fallback_source}] Error fetching albums for {artist.artist_name}: {e}")
|
|
|
|
# Process Spotify if authenticated
|
|
if spotify_available and artist.spotify_artist_id:
|
|
try:
|
|
albums = self.spotify_client.get_artist_albums(
|
|
artist.spotify_artist_id,
|
|
album_type='album,single,ep',
|
|
limit=20,
|
|
skip_cache=True
|
|
)
|
|
for album in albums or []:
|
|
process_album(album, artist.artist_name, artist.spotify_artist_id, fallback_id if fallback_source == 'itunes' else None, 'spotify')
|
|
except Exception as e:
|
|
logger.debug(f"Error fetching Spotify albums for {artist.artist_name}: {e}")
|
|
|
|
time.sleep(DELAY_BETWEEN_ARTISTS)
|
|
|
|
# Process similar artists
|
|
for artist in similar_artists:
|
|
# Always process fallback source (iTunes or Deezer) as baseline
|
|
fallback_id = artist.similar_artist_itunes_id if fallback_source == 'itunes' else getattr(artist, 'similar_artist_deezer_id', None)
|
|
if not fallback_id:
|
|
# Try to resolve fallback ID on-the-fly (with retry for rate limiting)
|
|
try:
|
|
results = itunes_api_call_with_retry(
|
|
itunes_client.search_artists, artist.similar_artist_name, limit=1
|
|
)
|
|
if results and len(results) > 0:
|
|
fallback_id = results[0].id
|
|
# Cache for future
|
|
if fallback_source == 'deezer':
|
|
self.database.update_similar_artist_deezer_id(artist.id, fallback_id)
|
|
else:
|
|
self.database.update_similar_artist_itunes_id(artist.id, fallback_id)
|
|
fallback_resolved += 1
|
|
logger.debug(f"[{fallback_source}] Resolved ID for similar artist {artist.similar_artist_name}: {fallback_id}")
|
|
else:
|
|
fallback_failed_resolve += 1
|
|
logger.info(f"[{fallback_source}] No artist found for similar: {artist.similar_artist_name}")
|
|
except Exception as e:
|
|
fallback_failed_resolve += 1
|
|
logger.info(f"[{fallback_source}] Failed to resolve similar {artist.similar_artist_name}: {e}")
|
|
|
|
if fallback_id:
|
|
try:
|
|
albums = itunes_api_call_with_retry(
|
|
itunes_client.get_artist_albums, fallback_id, album_type='album,single,ep', limit=20
|
|
)
|
|
for album in albums or []:
|
|
process_album(
|
|
album, artist.similar_artist_name, artist.similar_artist_spotify_id,
|
|
fallback_id if fallback_source == 'itunes' else None,
|
|
fallback_source,
|
|
artist_deezer_id=fallback_id if fallback_source == 'deezer' else None
|
|
)
|
|
except Exception as e:
|
|
logger.info(f"[{fallback_source}] Error fetching albums for similar {artist.similar_artist_name}: {e}")
|
|
|
|
# Process Spotify if authenticated
|
|
if spotify_available and artist.similar_artist_spotify_id:
|
|
try:
|
|
albums = self.spotify_client.get_artist_albums(
|
|
artist.similar_artist_spotify_id,
|
|
album_type='album,single,ep',
|
|
limit=20,
|
|
skip_cache=True
|
|
)
|
|
for album in albums or []:
|
|
process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, fallback_id if fallback_source == 'itunes' else None, 'spotify')
|
|
except Exception as e:
|
|
logger.debug(f"Error fetching Spotify albums for {artist.similar_artist_name}: {e}")
|
|
|
|
time.sleep(DELAY_BETWEEN_ARTISTS)
|
|
|
|
total_cached = cached_count['spotify'] + cached_count.get(fallback_source, 0)
|
|
logger.info(f"Cached {total_cached} recent albums (Spotify: {cached_count['spotify']}, {fallback_source}: {cached_count.get(fallback_source, 0)}) from {albums_checked} albums checked")
|
|
logger.info(f"[{fallback_source}] ID resolution stats: {fallback_resolved} resolved, {fallback_failed_resolve} failed")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error caching discovery recent albums: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
def _get_listening_profile(self, profile_id: int = 1) -> dict:
|
|
"""Build a listening profile from the user's play history for personalized discovery.
|
|
|
|
Returns a dict with top artists, genres, listening velocity, etc.
|
|
Falls back to empty/default values if no listening data exists.
|
|
"""
|
|
try:
|
|
stats = self.database.get_listening_stats('30d')
|
|
if not stats or stats.get('total_plays', 0) == 0:
|
|
return {'has_data': False, 'top_artist_names': set(), 'top_genres': set(),
|
|
'genre_weights': {}, 'artist_play_counts': {}, 'avg_daily_plays': 0, 'listening_diversity': 0}
|
|
|
|
top_artists = self.database.get_top_artists('30d', 20)
|
|
top_artist_names = {a['name'].lower() for a in top_artists}
|
|
|
|
# Build play count lookup for artist penalty scoring
|
|
artist_play_counts = {a['name'].lower(): a['play_count'] for a in top_artists}
|
|
|
|
genre_breakdown = self.database.get_genre_breakdown('30d')
|
|
top_genres = {g['genre'].lower() for g in genre_breakdown[:5]} if genre_breakdown else set()
|
|
genre_weights = {g['genre'].lower(): g['percentage'] for g in genre_breakdown} if genre_breakdown else {}
|
|
|
|
return {
|
|
'has_data': True,
|
|
'top_artist_names': top_artist_names,
|
|
'artist_play_counts': artist_play_counts,
|
|
'top_genres': top_genres,
|
|
'genre_weights': genre_weights,
|
|
'avg_daily_plays': stats.get('total_plays', 0) / 30,
|
|
'listening_diversity': stats.get('unique_artists', 0),
|
|
}
|
|
except Exception as e:
|
|
logger.debug(f"Could not build listening profile: {e}")
|
|
return {'has_data': False, 'top_artist_names': set(), 'top_genres': set(),
|
|
'genre_weights': {}, 'avg_daily_plays': 0, 'listening_diversity': 0}
|
|
|
|
def curate_discovery_playlists(self, profile_id: int = 1):
|
|
"""
|
|
Curate consistent playlist selections that stay the same until next discovery pool update.
|
|
|
|
Supports both Spotify and iTunes sources - creates separate curated playlists for each.
|
|
- Release Radar: Prioritizes freshness + popularity from recent releases
|
|
- Discovery Weekly: Balanced mix of popular picks, deep cuts, and mid-tier tracks
|
|
|
|
Uses listening stats (if available) to personalize scoring.
|
|
"""
|
|
try:
|
|
import random
|
|
from datetime import datetime
|
|
|
|
logger.info("Curating discovery playlists...")
|
|
|
|
# Build listening profile for personalization
|
|
profile = self._get_listening_profile(profile_id)
|
|
if profile['has_data']:
|
|
logger.info(f"Listening profile: {len(profile['top_artist_names'])} top artists, "
|
|
f"{len(profile['top_genres'])} top genres, "
|
|
f"{profile['avg_daily_plays']:.1f} avg daily plays")
|
|
|
|
# Determine available sources
|
|
spotify_available = self.spotify_client and self.spotify_client.is_spotify_authenticated()
|
|
itunes_client, fallback_source = _get_fallback_metadata_client()
|
|
|
|
# Process each available source
|
|
sources_to_process = [fallback_source] # Fallback source (iTunes/Deezer) always available
|
|
if spotify_available:
|
|
sources_to_process.append('spotify')
|
|
|
|
# Pre-build artist genre cache from local DB for genre affinity scoring
|
|
_artist_genre_cache = {}
|
|
if profile['has_data']:
|
|
try:
|
|
import json as _json
|
|
_conn = self.database._get_connection()
|
|
_cur = _conn.cursor()
|
|
_cur.execute("SELECT name, genres FROM artists WHERE genres IS NOT NULL AND genres != ''")
|
|
for _row in _cur.fetchall():
|
|
if not _row[0]:
|
|
continue
|
|
try:
|
|
_parsed = _json.loads(_row[1])
|
|
if isinstance(_parsed, list):
|
|
_artist_genre_cache[_row[0].lower()] = {g.lower() for g in _parsed if g}
|
|
except (ValueError, TypeError):
|
|
_artist_genre_cache[_row[0].lower()] = {g.strip().lower() for g in _row[1].split(',') if g.strip()}
|
|
_conn.close()
|
|
logger.debug(f"Built genre cache for {len(_artist_genre_cache)} artists")
|
|
except Exception:
|
|
pass
|
|
|
|
logger.info(f"Curating playlists for sources: {sources_to_process}")
|
|
|
|
for source in sources_to_process:
|
|
logger.info(f"Curating Release Radar for {source}...")
|
|
|
|
# 1. Curate Release Radar - 50 tracks from recent albums
|
|
recent_albums = self.database.get_discovery_recent_albums(limit=50, source=source, profile_id=profile_id)
|
|
release_radar_tracks = []
|
|
|
|
if not recent_albums:
|
|
logger.warning(f"[{source.upper()}] No recent albums found for Release Radar - check cache_discovery_recent_albums()")
|
|
|
|
if recent_albums:
|
|
# Group albums by artist for variety
|
|
albums_by_artist = {}
|
|
for album in recent_albums:
|
|
artist = album['artist_name']
|
|
if artist not in albums_by_artist:
|
|
albums_by_artist[artist] = []
|
|
albums_by_artist[artist].append(album)
|
|
|
|
# Get tracks from each album
|
|
artist_track_data = {}
|
|
|
|
for artist, albums in albums_by_artist.items():
|
|
artist_track_data[artist] = []
|
|
|
|
for album in albums:
|
|
try:
|
|
# Get album data from appropriate source
|
|
if source == 'spotify':
|
|
album_id = album.get('album_spotify_id')
|
|
elif source == 'deezer':
|
|
album_id = album.get('album_deezer_id')
|
|
else:
|
|
album_id = album.get('album_itunes_id')
|
|
if not album_id:
|
|
continue
|
|
|
|
if source == 'spotify':
|
|
album_data = self.spotify_client.get_album(album_id)
|
|
else:
|
|
album_data = itunes_api_call_with_retry(
|
|
itunes_client.get_album, album_id
|
|
)
|
|
|
|
if not album_data or 'tracks' not in album_data:
|
|
continue
|
|
|
|
# Calculate days since release for recency score
|
|
days_old = 14
|
|
try:
|
|
release_date_str = album.get('release_date', '')
|
|
if release_date_str and len(release_date_str) >= 10:
|
|
release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d")
|
|
days_old = (datetime.now() - release_date).days
|
|
except:
|
|
pass
|
|
|
|
for track in album_data['tracks'].get('items', []):
|
|
track_id = track.get('id')
|
|
if not track_id:
|
|
continue
|
|
|
|
# Calculate track score
|
|
recency_score = max(0, 100 - (days_old * 7))
|
|
popularity_score = track.get('popularity', album_data.get('popularity', 0))
|
|
# iTunes/Deezer have no popularity — use recency-based synthetic score
|
|
if source in ('itunes', 'deezer') and popularity_score == 0:
|
|
popularity_score = max(40, 70 - days_old)
|
|
is_single = album.get('album_type', 'album') == 'single'
|
|
single_bonus = 20 if is_single else 0
|
|
|
|
# Personalization bonuses (from listening profile)
|
|
genre_bonus = 0
|
|
artist_bonus = 0
|
|
overplay_penalty = 0
|
|
if profile['has_data']:
|
|
artist_lower = artist.lower()
|
|
# Genre affinity: check album/API genres, then use cached DB genres
|
|
artist_genres_lower = {g.lower() for g in (album.get('genres') or album_data.get('genres') or [])}
|
|
if not artist_genres_lower:
|
|
artist_genres_lower = _artist_genre_cache.get(artist_lower, set())
|
|
if artist_genres_lower & profile['top_genres']:
|
|
genre_bonus = 10
|
|
# Artist familiarity: boost tracks from artists user listens to
|
|
if artist_lower in profile['top_artist_names']:
|
|
artist_bonus = 15
|
|
# Overplay penalty: reduce score for artists user has heard too much
|
|
if profile['artist_play_counts'].get(artist_lower, 0) > 20:
|
|
overplay_penalty = -10
|
|
|
|
total_score = (recency_score * 0.45) + (popularity_score * 0.25) + single_bonus + genre_bonus + artist_bonus + overplay_penalty
|
|
|
|
full_track = {
|
|
'id': track_id,
|
|
'name': track.get('name', 'Unknown'),
|
|
'artists': track.get('artists', [{'name': artist}]),
|
|
'album': {
|
|
'id': album_data.get('id', ''),
|
|
'name': album_data.get('name', 'Unknown Album'),
|
|
'images': album_data.get('images', []),
|
|
'release_date': album_data.get('release_date', ''),
|
|
'album_type': album_data.get('album_type', 'album'),
|
|
},
|
|
'duration_ms': track.get('duration_ms', 0),
|
|
'popularity': popularity_score,
|
|
'score': total_score,
|
|
'source': source
|
|
}
|
|
artist_track_data[artist].append(full_track)
|
|
|
|
except Exception as e:
|
|
logger.debug(f"Error processing album for {artist}: {e}")
|
|
continue
|
|
|
|
# Balance by artist - max 6 tracks per artist
|
|
balanced_track_data = []
|
|
for artist, tracks in artist_track_data.items():
|
|
sorted_tracks = sorted(tracks, key=lambda t: t['score'], reverse=True)
|
|
balanced_track_data.extend(sorted_tracks[:6])
|
|
|
|
# Sort by score and shuffle
|
|
balanced_track_data.sort(key=lambda t: t['score'], reverse=True)
|
|
top_tracks = balanced_track_data[:75]
|
|
random.shuffle(top_tracks)
|
|
|
|
# Take final 50 tracks
|
|
release_radar_tracks = [track['id'] for track in top_tracks[:50]]
|
|
|
|
# Add tracks to discovery pool
|
|
for track_data in top_tracks[:50]:
|
|
try:
|
|
artist_name = track_data['artists'][0].get('name', 'Unknown') if track_data['artists'] else 'Unknown'
|
|
formatted_track = {
|
|
'track_name': track_data['name'],
|
|
'artist_name': artist_name,
|
|
'album_name': track_data['album'].get('name', 'Unknown'),
|
|
'album_cover_url': track_data['album']['images'][0]['url'] if track_data['album'].get('images') else None,
|
|
'duration_ms': track_data.get('duration_ms', 0),
|
|
'popularity': track_data.get('popularity', 0),
|
|
'release_date': track_data['album'].get('release_date', ''),
|
|
'is_new_release': True,
|
|
'track_data_json': track_data,
|
|
'artist_genres': []
|
|
}
|
|
if source == 'spotify':
|
|
formatted_track['spotify_track_id'] = track_data['id']
|
|
formatted_track['spotify_album_id'] = track_data['album'].get('id', '')
|
|
elif source == 'deezer':
|
|
formatted_track['deezer_track_id'] = track_data['id']
|
|
formatted_track['deezer_album_id'] = track_data['album'].get('id', '')
|
|
else:
|
|
formatted_track['itunes_track_id'] = track_data['id']
|
|
formatted_track['itunes_album_id'] = track_data['album'].get('id', '')
|
|
|
|
self.database.add_to_discovery_pool(formatted_track, source=source, profile_id=profile_id)
|
|
except Exception as e:
|
|
continue
|
|
|
|
# Save with source suffix for multi-source support
|
|
playlist_key = f'release_radar_{source}'
|
|
self.database.save_curated_playlist(playlist_key, release_radar_tracks, profile_id=profile_id)
|
|
logger.info(f"Release Radar ({source}) curated: {len(release_radar_tracks)} tracks")
|
|
|
|
# 2. Curate Discovery Weekly - 50 tracks from discovery pool
|
|
logger.info(f"Curating Discovery Weekly for {source}...")
|
|
discovery_tracks = self.database.get_discovery_pool_tracks(limit=2000, new_releases_only=False, source=source, profile_id=profile_id)
|
|
|
|
if not discovery_tracks:
|
|
logger.warning(f"[{source.upper()}] No discovery pool tracks found for Discovery Weekly - check populate_discovery_pool()")
|
|
|
|
discovery_weekly_tracks = []
|
|
if discovery_tracks:
|
|
# Separate tracks by popularity tiers
|
|
popular_picks = []
|
|
balanced_mix = []
|
|
deep_cuts = []
|
|
|
|
for track in discovery_tracks:
|
|
popularity = track.popularity if hasattr(track, 'popularity') else 50
|
|
if popularity >= 60:
|
|
popular_picks.append(track)
|
|
elif popularity >= 40:
|
|
balanced_mix.append(track)
|
|
else:
|
|
deep_cuts.append(track)
|
|
|
|
logger.info(f"Discovery pool ({source}): {len(popular_picks)} popular, {len(balanced_mix)} mid-tier, {len(deep_cuts)} deep cuts")
|
|
|
|
# Serendipity-weighted selection within each tier
|
|
def _serendipity_sort(tracks_list):
|
|
"""Sort by serendipity: prefer unknown artists in genres user likes."""
|
|
if not profile['has_data']:
|
|
random.shuffle(tracks_list)
|
|
return tracks_list
|
|
|
|
for t in tracks_list:
|
|
score = 1.0
|
|
t_artist = (t.artist_name or '').lower()
|
|
t_genres = _artist_genre_cache.get(t_artist, set())
|
|
|
|
# Boost artists user has NEVER played (true discovery)
|
|
if t_artist not in profile['top_artist_names']:
|
|
score += 0.5
|
|
# Boost genres user likes but hasn't explored
|
|
if t_genres & profile['top_genres']:
|
|
score += 0.3
|
|
# Penalize artists user already plays heavily
|
|
if profile['artist_play_counts'].get(t_artist, 0) > 10:
|
|
score -= 0.4
|
|
|
|
t._serendipity = score + random.random() * 0.2 # Small random factor
|
|
|
|
tracks_list.sort(key=lambda t: getattr(t, '_serendipity', 1.0), reverse=True)
|
|
return tracks_list
|
|
|
|
_serendipity_sort(popular_picks)
|
|
_serendipity_sort(balanced_mix)
|
|
_serendipity_sort(deep_cuts)
|
|
|
|
selected_tracks = []
|
|
selected_tracks.extend(popular_picks[:20])
|
|
selected_tracks.extend(balanced_mix[:20])
|
|
selected_tracks.extend(deep_cuts[:10])
|
|
random.shuffle(selected_tracks)
|
|
|
|
# Extract appropriate track IDs based on source
|
|
for track in selected_tracks:
|
|
if source == 'spotify' and track.spotify_track_id:
|
|
discovery_weekly_tracks.append(track.spotify_track_id)
|
|
elif source == 'itunes' and track.itunes_track_id:
|
|
discovery_weekly_tracks.append(track.itunes_track_id)
|
|
elif source == 'deezer' and track.deezer_track_id:
|
|
discovery_weekly_tracks.append(track.deezer_track_id)
|
|
|
|
playlist_key = f'discovery_weekly_{source}'
|
|
self.database.save_curated_playlist(playlist_key, discovery_weekly_tracks, profile_id=profile_id)
|
|
logger.info(f"Discovery Weekly ({source}) curated: {len(discovery_weekly_tracks)} tracks")
|
|
|
|
# 3. "Because You Listen To" — personalized sections based on top played artists
|
|
if profile['has_data']:
|
|
logger.info("Building 'Because You Listen To' playlists...")
|
|
top_played = self.database.get_top_artists('30d', 3)
|
|
active_source_for_bylt = 'spotify' if spotify_available else fallback_source
|
|
all_pool_tracks = self.database.get_discovery_pool_tracks(
|
|
limit=2000, new_releases_only=False,
|
|
source=active_source_for_bylt, profile_id=profile_id
|
|
)
|
|
|
|
# Build source_artist_id → artist_name mapping from watchlist
|
|
_wa_id_to_name = {}
|
|
try:
|
|
_wa_list = self.database.get_watchlist_artists(profile_id=profile_id)
|
|
for _wa in _wa_list:
|
|
_wa_id_to_name[str(_wa.id)] = (_wa.artist_name or '').lower()
|
|
except Exception:
|
|
pass
|
|
|
|
all_similar = self.database.get_top_similar_artists(limit=200, profile_id=profile_id)
|
|
|
|
for i, played_artist in enumerate(top_played):
|
|
try:
|
|
artist_name = played_artist['name']
|
|
artist_lower = artist_name.lower()
|
|
|
|
# Find similar artists to this played artist via the similar_artists table
|
|
similar_names = set()
|
|
for s in all_similar:
|
|
# Check if this similar artist's source matches our played artist
|
|
src_id = str(getattr(s, 'source_artist_id', ''))
|
|
src_name = _wa_id_to_name.get(src_id, '')
|
|
sim_name = getattr(s, 'similar_artist_name', '') or ''
|
|
if src_name == artist_lower and sim_name:
|
|
similar_names.add(sim_name.lower())
|
|
|
|
if not similar_names:
|
|
# Fallback: find pool tracks from same genre
|
|
played_genres = _artist_genre_cache.get(artist_lower, set())
|
|
if played_genres:
|
|
for t in all_pool_tracks:
|
|
t_artist_lower = (t.artist_name or '').lower()
|
|
if t_artist_lower != artist_lower and _artist_genre_cache.get(t_artist_lower, set()) & played_genres:
|
|
similar_names.add(t_artist_lower)
|
|
if len(similar_names) >= 20:
|
|
break
|
|
|
|
if not similar_names:
|
|
continue
|
|
|
|
# Pick tracks from those similar artists in the pool
|
|
matching_tracks = []
|
|
for t in all_pool_tracks:
|
|
if (t.artist_name or '').lower() in similar_names:
|
|
if active_source_for_bylt == 'spotify' and t.spotify_track_id:
|
|
matching_tracks.append(t.spotify_track_id)
|
|
elif active_source_for_bylt == 'itunes' and t.itunes_track_id:
|
|
matching_tracks.append(t.itunes_track_id)
|
|
elif active_source_for_bylt == 'deezer' and t.deezer_track_id:
|
|
matching_tracks.append(t.deezer_track_id)
|
|
|
|
if len(matching_tracks) >= 15:
|
|
break
|
|
|
|
if matching_tracks:
|
|
import random as _rnd
|
|
_rnd.shuffle(matching_tracks)
|
|
playlist_key = f'because_you_listen_to_{i}'
|
|
self.database.save_curated_playlist(playlist_key, matching_tracks[:10], profile_id=profile_id)
|
|
# Store the source artist name in metadata
|
|
self.database.set_metadata(f'bylt_artist_{i}', artist_name)
|
|
logger.info(f"'Because You Listen To {artist_name}': {len(matching_tracks[:10])} tracks")
|
|
except Exception as e:
|
|
logger.debug(f"Error building BYLT for {played_artist.get('name', '?')}: {e}")
|
|
|
|
# Also save without suffix for backward compatibility (use active source)
|
|
active_source = 'spotify' if spotify_available else fallback_source
|
|
release_radar_key = f'release_radar_{active_source}'
|
|
discovery_weekly_key = f'discovery_weekly_{active_source}'
|
|
|
|
# Copy active source playlists to non-suffixed keys
|
|
release_radar_ids = self.database.get_curated_playlist(release_radar_key, profile_id=profile_id) or []
|
|
discovery_weekly_ids = self.database.get_curated_playlist(discovery_weekly_key, profile_id=profile_id) or []
|
|
self.database.save_curated_playlist('release_radar', release_radar_ids, profile_id=profile_id)
|
|
self.database.save_curated_playlist('discovery_weekly', discovery_weekly_ids, profile_id=profile_id)
|
|
|
|
logger.info("Playlist curation complete")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error curating discovery playlists: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
def sync_spotify_library_cache(self, profile_id=1):
|
|
"""Sync user's saved Spotify albums into the local cache.
|
|
|
|
Runs after the main watchlist scan. First sync fetches all saved albums;
|
|
subsequent syncs are incremental (only fetch newly saved albums).
|
|
Every 7 days, does a full re-sync to detect un-saved albums.
|
|
"""
|
|
if not self.spotify_client or not self.spotify_client.is_spotify_authenticated():
|
|
logger.debug("Spotify not authenticated, skipping library cache sync")
|
|
return
|
|
|
|
logger.info("📚 Syncing Spotify library cache...")
|
|
|
|
try:
|
|
last_sync = self.database.get_metadata('spotify_library_last_sync')
|
|
last_full_sync = self.database.get_metadata('spotify_library_last_full_sync')
|
|
|
|
# Determine if we need a full sync (first time or every 7 days)
|
|
do_full_sync = False
|
|
if not last_sync:
|
|
do_full_sync = True
|
|
logger.info("First-time Spotify library sync — fetching all saved albums")
|
|
elif not last_full_sync:
|
|
# last_sync exists but last_full_sync doesn't — first run with this code
|
|
do_full_sync = True
|
|
logger.info("Full re-sync triggered (no full sync recorded)")
|
|
else:
|
|
try:
|
|
last_full_dt = datetime.fromisoformat(last_full_sync)
|
|
if datetime.now() - last_full_dt > timedelta(days=7):
|
|
do_full_sync = True
|
|
logger.info("Full re-sync triggered (>7 days since last full sync)")
|
|
except (ValueError, TypeError):
|
|
do_full_sync = True
|
|
|
|
# Fetch albums from Spotify
|
|
since_timestamp = None if do_full_sync else last_sync
|
|
albums = self.spotify_client.get_saved_albums(since_timestamp=since_timestamp)
|
|
|
|
if not albums and not do_full_sync:
|
|
logger.info("No new saved albums since last sync")
|
|
return
|
|
|
|
if albums:
|
|
self.database.upsert_spotify_library_albums(albums, profile_id=profile_id)
|
|
|
|
# On full sync, remove albums that are no longer saved
|
|
if do_full_sync and albums:
|
|
fetched_ids = {a['spotify_album_id'] for a in albums}
|
|
self.database.remove_spotify_library_albums_not_in(fetched_ids, profile_id=profile_id)
|
|
self.database.set_metadata('spotify_library_last_full_sync', datetime.now().isoformat())
|
|
|
|
# Update last sync timestamp
|
|
self.database.set_metadata('spotify_library_last_sync', datetime.now().isoformat())
|
|
|
|
logger.info(f"✅ Spotify library cache sync complete — {len(albums)} albums processed")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error syncing Spotify library cache: {e}")
|
|
|
|
def _populate_seasonal_content(self):
|
|
"""
|
|
Populate seasonal content as part of watchlist scan.
|
|
|
|
IMPROVED: Integrated with discovery system
|
|
- Checks if seasonal content needs update (7-day threshold)
|
|
- Populates content for all seasons
|
|
- Curates seasonal playlists
|
|
- Runs once per week automatically
|
|
"""
|
|
try:
|
|
from core.seasonal_discovery import get_seasonal_discovery_service
|
|
|
|
logger.info("Checking seasonal content update...")
|
|
|
|
seasonal_service = get_seasonal_discovery_service(self.spotify_client, self.database)
|
|
|
|
# Get current season to prioritize
|
|
current_season = seasonal_service.get_current_season()
|
|
|
|
if current_season:
|
|
# Always update current season if needed
|
|
if seasonal_service.should_populate_seasonal_content(current_season, days_threshold=7):
|
|
logger.info(f"Populating current season: {current_season}")
|
|
seasonal_service.populate_seasonal_content(current_season)
|
|
seasonal_service.curate_seasonal_playlist(current_season)
|
|
else:
|
|
logger.info(f"Current season '{current_season}' is up to date")
|
|
|
|
# Update other seasons in background (less frequently - 14 day threshold)
|
|
from core.seasonal_discovery import SEASONAL_CONFIG
|
|
for season_key in SEASONAL_CONFIG.keys():
|
|
if season_key == current_season:
|
|
continue # Already handled above
|
|
|
|
if seasonal_service.should_populate_seasonal_content(season_key, days_threshold=14):
|
|
logger.info(f"Populating season: {season_key}")
|
|
seasonal_service.populate_seasonal_content(season_key)
|
|
seasonal_service.curate_seasonal_playlist(season_key)
|
|
|
|
logger.info("Seasonal content update complete")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error populating seasonal content: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
# Singleton instance
|
|
_watchlist_scanner_instance = None
|
|
|
|
def get_watchlist_scanner(spotify_client: SpotifyClient) -> WatchlistScanner:
|
|
"""Get the global watchlist scanner instance"""
|
|
global _watchlist_scanner_instance
|
|
if _watchlist_scanner_instance is None:
|
|
_watchlist_scanner_instance = WatchlistScanner(spotify_client)
|
|
return _watchlist_scanner_instance |