From 3cb88669e3e7d6e7d5aa6278768aca93ab2f4a47 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Sat, 24 Jan 2026 21:48:17 -0800 Subject: [PATCH] Fix iTunes-only Discover page not loading data - Add similar artists fetching to web UI scan loop - Add database migration for UNIQUE constraint on similar_artists table - Add source-agnostic /api/discover/album endpoint for iTunes support - Fix NOT NULL constraint on discovery_recent_albums blocking iTunes albums - Add fallback to watchlist artists when no similar artists exist - Add /api/discover/refresh and /api/discover/diagnose endpoints - Add retry logic with exponential backoff for iTunes API calls - Ensure cache_discovery_recent_albums runs even when pool population skips --- core/watchlist_scanner.py | 147 ++++++++++++-- database/music_database.py | 84 ++++++++ tools/diagnose_itunes_discover.py | 270 +++++++++++++++++++++++++ web_server.py | 320 +++++++++++++++++++++++++++++- webui/static/script.js | 21 +- 5 files changed, 808 insertions(+), 34 deletions(-) create mode 100644 tools/diagnose_itunes_discover.py diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index db7c9b17..367ebddf 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -21,9 +21,61 @@ logger = get_logger("watchlist_scanner") # Rate limiting constants for watchlist operations DELAY_BETWEEN_ARTISTS = 2.0 # 2 seconds between different artists -DELAY_BETWEEN_ALBUMS = 0.5 # 500ms between albums for same artist +DELAY_BETWEEN_ALBUMS = 0.5 # 500ms between albums for same artist DELAY_BETWEEN_API_BATCHES = 1.0 # 1 second between API batch operations +# iTunes API retry configuration +ITUNES_MAX_RETRIES = 3 +ITUNES_BASE_DELAY = 1.0 # Base delay in seconds for exponential backoff + + +def itunes_api_call_with_retry(func, *args, max_retries=ITUNES_MAX_RETRIES, **kwargs): + """ + Execute an iTunes API call with exponential backoff retry logic. + + Args: + func: The function to call + *args: Arguments to pass to the function + max_retries: Maximum number of retry attempts + **kwargs: Keyword arguments to pass to the function + + Returns: + The result of the function call, or None if all retries failed + """ + last_error = None + for attempt in range(max_retries): + try: + result = func(*args, **kwargs) + return result + except requests.exceptions.HTTPError as e: + # Handle rate limiting (429) and server errors (5xx) + if e.response is not None and e.response.status_code == 429: + delay = ITUNES_BASE_DELAY * (2 ** attempt) + logger.warning(f"[iTunes] Rate limited, retrying in {delay}s (attempt {attempt + 1}/{max_retries})") + time.sleep(delay) + last_error = e + elif e.response is not None and e.response.status_code >= 500: + delay = ITUNES_BASE_DELAY * (2 ** attempt) + logger.warning(f"[iTunes] Server error {e.response.status_code}, retrying in {delay}s (attempt {attempt + 1}/{max_retries})") + time.sleep(delay) + last_error = e + else: + raise # Don't retry on client errors (4xx except 429) + except requests.exceptions.RequestException as e: + # Retry on connection errors + delay = ITUNES_BASE_DELAY * (2 ** attempt) + logger.warning(f"[iTunes] Connection error, retrying in {delay}s (attempt {attempt + 1}/{max_retries}): {e}") + time.sleep(delay) + last_error = e + except Exception as e: + # Don't retry on other exceptions + raise + + if last_error: + logger.error(f"[iTunes] All {max_retries} retry attempts failed: {last_error}") + return None + + def clean_track_name_for_search(track_name): """ Intelligently cleans a track name for searching by removing noise while preserving important version information. @@ -1287,9 +1339,11 @@ class WatchlistScanner: except Exception as e: logger.debug(f"Spotify match failed for {artist_name_to_match}: {e}") - # Try to match on iTunes + # Try to match on iTunes (with retry for rate limiting) try: - itunes_results = itunes_client.search_artists(artist_name_to_match, limit=1) + itunes_results = itunes_api_call_with_retry( + itunes_client.search_artists, artist_name_to_match, limit=1 + ) if itunes_results and len(itunes_results) > 0: itunes_artist = itunes_results[0] # Skip if this is the searched artist @@ -1301,8 +1355,10 @@ class WatchlistScanner: # Use iTunes genres if we don't have Spotify genres if not artist_data['genres'] and hasattr(itunes_artist, 'genres'): artist_data['genres'] = itunes_artist.genres + else: + logger.info(f" [iTunes] No match found for: {artist_name_to_match}") except Exception as e: - logger.debug(f"iTunes match failed for {artist_name_to_match}: {e}") + logger.info(f" [iTunes] Match failed for {artist_name_to_match}: {e}") # Only add if we got at least one ID if artist_data['spotify_id'] or artist_data['itunes_id']: @@ -1314,7 +1370,11 @@ class WatchlistScanner: logger.debug(f"Error matching {artist_name_to_match}: {match_error}") continue - logger.info(f"Matched {len(matched_artists)} similar artists (Spotify + iTunes)") + # Log detailed matching statistics + itunes_matched = sum(1 for a in matched_artists if a.get('itunes_id')) + spotify_matched = sum(1 for a in matched_artists if a.get('spotify_id')) + both_matched = sum(1 for a in matched_artists if a.get('itunes_id') and a.get('spotify_id')) + logger.info(f"Matched {len(matched_artists)} similar artists - iTunes: {itunes_matched}, Spotify: {spotify_matched}, Both: {both_matched}") return matched_artists except requests.exceptions.RequestException as e: @@ -1436,9 +1496,15 @@ class WatchlistScanner: from datetime import datetime, timedelta import random - # Check if we should run (prevents over-polling) - if not self.database.should_populate_discovery_pool(hours_threshold=24): - logger.info("Discovery pool was populated recently (< 24 hours ago). Skipping.") + # Check if we should run discovery pool population (prevents over-polling) + skip_pool_population = not self.database.should_populate_discovery_pool(hours_threshold=24) + + if skip_pool_population: + logger.info("Discovery pool was populated recently (< 24 hours ago). Skipping pool population.") + logger.info("But still refreshing recent albums cache and curated playlists...") + # Still run these even when skipping main pool population + self.cache_discovery_recent_albums() + self.curate_discovery_playlists() return logger.info("Populating discovery pool from similar artists...") @@ -1461,7 +1527,11 @@ class WatchlistScanner: similar_artists = self.database.get_top_similar_artists(limit=top_artists_limit) if not similar_artists: - logger.info("No similar artists found to populate discovery pool") + logger.info("No similar artists found to populate discovery pool from similar artists") + logger.info("But still caching recent albums from watchlist artists and curating playlists...") + # Still run these even without similar artists - they use watchlist artists + self.cache_discovery_recent_albums() + self.curate_discovery_playlists() return logger.info(f"Processing {len(similar_artists)} top similar artists for discovery pool") @@ -2056,26 +2126,40 @@ class WatchlistScanner: logger.debug(f"Error processing album: {e}") return False + # Track resolution stats + itunes_resolved = 0 + itunes_failed_resolve = 0 + # Process watchlist artists for artist in watchlist_artists: # Always process iTunes (baseline) itunes_id = artist.itunes_artist_id if not itunes_id: - # Try to resolve iTunes ID on-the-fly + # Try to resolve iTunes ID on-the-fly (with retry for rate limiting) try: - results = itunes_client.search_artists(artist.artist_name, limit=1) + results = itunes_api_call_with_retry( + itunes_client.search_artists, artist.artist_name, limit=1 + ) if results and len(results) > 0: itunes_id = results[0].id - except: - pass + itunes_resolved += 1 + logger.debug(f"[iTunes] Resolved ID for {artist.artist_name}: {itunes_id}") + else: + itunes_failed_resolve += 1 + logger.info(f"[iTunes] No artist found for: {artist.artist_name}") + except Exception as e: + itunes_failed_resolve += 1 + logger.info(f"[iTunes] Failed to resolve {artist.artist_name}: {e}") if itunes_id: try: - albums = itunes_client.get_artist_albums(itunes_id, album_type='album,single', limit=20) + albums = itunes_api_call_with_retry( + itunes_client.get_artist_albums, itunes_id, album_type='album,single', limit=20 + ) for album in albums or []: process_album(album, artist.artist_name, artist.spotify_artist_id, itunes_id, 'itunes') except Exception as e: - logger.debug(f"Error fetching iTunes albums for {artist.artist_name}: {e}") + logger.info(f"[iTunes] Error fetching albums for {artist.artist_name}: {e}") # Process Spotify if authenticated if spotify_available and artist.spotify_artist_id: @@ -2097,23 +2181,33 @@ class WatchlistScanner: # Always process iTunes (baseline) itunes_id = artist.similar_artist_itunes_id if not itunes_id: - # Try to resolve iTunes ID on-the-fly + # Try to resolve iTunes ID on-the-fly (with retry for rate limiting) try: - results = itunes_client.search_artists(artist.similar_artist_name, limit=1) + results = itunes_api_call_with_retry( + itunes_client.search_artists, artist.similar_artist_name, limit=1 + ) if results and len(results) > 0: itunes_id = results[0].id # Cache for future self.database.update_similar_artist_itunes_id(artist.id, itunes_id) - except: - pass + itunes_resolved += 1 + logger.debug(f"[iTunes] Resolved ID for similar artist {artist.similar_artist_name}: {itunes_id}") + else: + itunes_failed_resolve += 1 + logger.info(f"[iTunes] No artist found for similar: {artist.similar_artist_name}") + except Exception as e: + itunes_failed_resolve += 1 + logger.info(f"[iTunes] Failed to resolve similar {artist.similar_artist_name}: {e}") if itunes_id: try: - albums = itunes_client.get_artist_albums(itunes_id, album_type='album,single', limit=20) + albums = itunes_api_call_with_retry( + itunes_client.get_artist_albums, itunes_id, album_type='album,single', limit=20 + ) for album in albums or []: process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, itunes_id, 'itunes') except Exception as e: - logger.debug(f"Error fetching iTunes albums for {artist.similar_artist_name}: {e}") + logger.info(f"[iTunes] Error fetching albums for similar {artist.similar_artist_name}: {e}") # Process Spotify if authenticated if spotify_available and artist.similar_artist_spotify_id: @@ -2132,6 +2226,7 @@ class WatchlistScanner: total_cached = cached_count['spotify'] + cached_count['itunes'] logger.info(f"Cached {total_cached} recent albums (Spotify: {cached_count['spotify']}, iTunes: {cached_count['itunes']}) from {albums_checked} albums checked") + logger.info(f"[iTunes] ID resolution stats: {itunes_resolved} resolved, {itunes_failed_resolve} failed") except Exception as e: logger.error(f"Error caching discovery recent albums: {e}") @@ -2171,6 +2266,9 @@ class WatchlistScanner: recent_albums = self.database.get_discovery_recent_albums(limit=50, source=source) release_radar_tracks = [] + if not recent_albums: + logger.warning(f"[{source.upper()}] No recent albums found for Release Radar - check cache_discovery_recent_albums()") + if recent_albums: # Group albums by artist for variety albums_by_artist = {} @@ -2196,7 +2294,9 @@ class WatchlistScanner: if source == 'spotify': album_data = self.spotify_client.get_album(album_id) else: - album_data = itunes_client.get_album(album_id) + album_data = itunes_api_call_with_retry( + itunes_client.get_album, album_id + ) if not album_data or 'tracks' not in album_data: continue @@ -2295,6 +2395,9 @@ class WatchlistScanner: logger.info(f"Curating Discovery Weekly for {source}...") discovery_tracks = self.database.get_discovery_pool_tracks(limit=2000, new_releases_only=False, source=source) + if not discovery_tracks: + logger.warning(f"[{source.upper()}] No discovery pool tracks found for Discovery Weekly - check populate_discovery_pool()") + discovery_weekly_tracks = [] if discovery_tracks: # Separate tracks by popularity tiers diff --git a/database/music_database.py b/database/music_database.py index 9a08b81b..2645e2ec 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -636,6 +636,90 @@ class MusicDatabase: cursor.execute("ALTER TABLE discovery_recent_albums ADD COLUMN source TEXT DEFAULT 'spotify'") logger.info("Added iTunes columns to discovery_recent_albums table for dual-source discovery") + # Migration: Fix NOT NULL constraint on album_spotify_id (required for iTunes-only albums) + # Check if album_spotify_id has NOT NULL constraint by checking table schema + cursor.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='discovery_recent_albums'") + table_schema = cursor.fetchone() + if table_schema and 'album_spotify_id TEXT NOT NULL' in (table_schema[0] or ''): + logger.info("Migrating discovery_recent_albums to allow NULL album_spotify_id for iTunes support...") + # SQLite doesn't support ALTER COLUMN, so recreate table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS discovery_recent_albums_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + album_spotify_id TEXT, + album_itunes_id TEXT, + artist_spotify_id TEXT, + artist_itunes_id TEXT, + source TEXT NOT NULL DEFAULT 'spotify', + album_name TEXT NOT NULL, + artist_name TEXT NOT NULL, + album_cover_url TEXT, + release_date TEXT NOT NULL, + album_type TEXT DEFAULT 'album', + cached_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(album_spotify_id, album_itunes_id, source) + ) + """) + cursor.execute(""" + INSERT OR IGNORE INTO discovery_recent_albums_new + SELECT * FROM discovery_recent_albums + """) + cursor.execute("DROP TABLE discovery_recent_albums") + cursor.execute("ALTER TABLE discovery_recent_albums_new RENAME TO discovery_recent_albums") + conn.commit() + logger.info("Successfully migrated discovery_recent_albums table for iTunes support") + + # Migration: Add UNIQUE constraint to similar_artists table + # Test if ON CONFLICT works by trying a dummy operation + needs_similar_migration = False + try: + cursor.execute(""" + INSERT INTO similar_artists + (source_artist_id, similar_artist_name, similarity_rank, occurrence_count, last_updated) + VALUES ('__migration_test__', '__migration_test__', 1, 1, CURRENT_TIMESTAMP) + ON CONFLICT(source_artist_id, similar_artist_name) + DO UPDATE SET occurrence_count = occurrence_count + """) + # Clean up test row + cursor.execute("DELETE FROM similar_artists WHERE source_artist_id = '__migration_test__'") + logger.info("similar_artists table has correct UNIQUE constraint") + except Exception as constraint_error: + logger.info(f"similar_artists needs migration (constraint test failed: {constraint_error})") + needs_similar_migration = True + + if needs_similar_migration: + logger.info("Migrating similar_artists to add UNIQUE constraint...") + # Get a fresh connection for the migration + with self._get_connection() as migration_conn: + migration_cursor = migration_conn.cursor() + # SQLite doesn't support adding constraints, so recreate table + migration_cursor.execute("DROP TABLE IF EXISTS similar_artists_new") + migration_cursor.execute(""" + CREATE TABLE similar_artists_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_artist_id TEXT NOT NULL, + similar_artist_spotify_id TEXT, + similar_artist_itunes_id TEXT, + similar_artist_name TEXT NOT NULL, + similarity_rank INTEGER DEFAULT 1, + occurrence_count INTEGER DEFAULT 1, + last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(source_artist_id, similar_artist_name) + ) + """) + migration_cursor.execute(""" + INSERT OR IGNORE INTO similar_artists_new + (source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, + similar_artist_name, similarity_rank, occurrence_count, last_updated) + SELECT source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id, + similar_artist_name, similarity_rank, occurrence_count, last_updated + FROM similar_artists + """) + migration_cursor.execute("DROP TABLE similar_artists") + migration_cursor.execute("ALTER TABLE similar_artists_new RENAME TO similar_artists") + migration_conn.commit() + logger.info("Successfully migrated similar_artists table with UNIQUE constraint") + # ============== INDEXES (after migrations to ensure columns exist) ============== cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_source ON similar_artists (source_artist_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_spotify ON similar_artists (similar_artist_spotify_id)") diff --git a/tools/diagnose_itunes_discover.py b/tools/diagnose_itunes_discover.py new file mode 100644 index 00000000..f3b0a15b --- /dev/null +++ b/tools/diagnose_itunes_discover.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +""" +Diagnostic script to check iTunes data availability for the Discover page. + +Run this script to identify issues with iTunes data population: +- Similar artists missing iTunes IDs +- Discovery pool tracks by source +- Recent albums by source +- Curated playlists status + +Usage: + python tools/diagnose_itunes_discover.py +""" + +import sys +import os +import json + +# Add parent directory to path for imports +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from database.music_database import MusicDatabase + + +def diagnose_itunes_discover(): + """Run diagnostic checks for iTunes discover data.""" + + print("=" * 60) + print("iTunes Discover Page Diagnostic Report") + print("=" * 60) + + db = MusicDatabase() + + # 1. Check Similar Artists + print("\n[1] SIMILAR ARTISTS") + print("-" * 40) + + try: + with db._get_connection() as conn: + cursor = conn.cursor() + + # Total similar artists + cursor.execute("SELECT COUNT(*) as total FROM similar_artists") + total = cursor.fetchone()['total'] + + # With iTunes IDs + cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_itunes_id IS NOT NULL") + with_itunes = cursor.fetchone()['count'] + + # With Spotify IDs + cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_spotify_id IS NOT NULL") + with_spotify = cursor.fetchone()['count'] + + # With both + cursor.execute(""" + SELECT COUNT(*) as count FROM similar_artists + WHERE similar_artist_itunes_id IS NOT NULL + AND similar_artist_spotify_id IS NOT NULL + """) + with_both = cursor.fetchone()['count'] + + print(f" Total similar artists: {total}") + print(f" With iTunes ID: {with_itunes} ({100*with_itunes/total:.1f}%)" if total > 0 else " With iTunes ID: 0") + print(f" With Spotify ID: {with_spotify} ({100*with_spotify/total:.1f}%)" if total > 0 else " With Spotify ID: 0") + print(f" With BOTH IDs: {with_both} ({100*with_both/total:.1f}%)" if total > 0 else " With BOTH IDs: 0") + + if with_itunes == 0 and total > 0: + print(" [CRITICAL] No similar artists have iTunes IDs - Hero section will be empty!") + elif with_itunes < total * 0.5: + print(" [WARNING] Less than 50% of similar artists have iTunes IDs") + else: + print(" [OK] iTunes coverage is adequate") + + except Exception as e: + print(f" [ERROR] Could not check similar artists: {e}") + + # 2. Check Discovery Pool + print("\n[2] DISCOVERY POOL") + print("-" * 40) + + try: + with db._get_connection() as conn: + cursor = conn.cursor() + + # Total tracks + cursor.execute("SELECT COUNT(*) as total FROM discovery_pool") + total = cursor.fetchone()['total'] + + # By source + cursor.execute(""" + SELECT source, COUNT(*) as count + FROM discovery_pool + GROUP BY source + """) + source_counts = {row['source']: row['count'] for row in cursor.fetchall()} + + print(f" Total tracks: {total}") + print(f" Spotify tracks: {source_counts.get('spotify', 0)}") + print(f" iTunes tracks: {source_counts.get('itunes', 0)}") + + if source_counts.get('itunes', 0) == 0 and total > 0: + print(" [CRITICAL] No iTunes tracks in discovery pool - Fresh Tape/Archives will be empty!") + elif source_counts.get('itunes', 0) < total * 0.3: + print(" [WARNING] Low iTunes track count in discovery pool") + else: + print(" [OK] iTunes tracks present") + + except Exception as e: + print(f" [ERROR] Could not check discovery pool: {e}") + + # 3. Check Recent Albums + print("\n[3] RECENT ALBUMS CACHE") + print("-" * 40) + + try: + with db._get_connection() as conn: + cursor = conn.cursor() + + # Total albums + cursor.execute("SELECT COUNT(*) as total FROM discovery_recent_albums") + total = cursor.fetchone()['total'] + + # By source + cursor.execute(""" + SELECT source, COUNT(*) as count + FROM discovery_recent_albums + GROUP BY source + """) + source_counts = {row['source']: row['count'] for row in cursor.fetchall()} + + print(f" Total recent albums: {total}") + print(f" Spotify albums: {source_counts.get('spotify', 0)}") + print(f" iTunes albums: {source_counts.get('itunes', 0)}") + + if source_counts.get('itunes', 0) == 0 and total > 0: + print(" [CRITICAL] No iTunes albums cached - Recent Releases section will be empty!") + elif source_counts.get('itunes', 0) < 5: + print(" [WARNING] Very few iTunes albums cached") + else: + print(" [OK] iTunes albums cached") + + except Exception as e: + print(f" [ERROR] Could not check recent albums: {e}") + + # 4. Check Curated Playlists + print("\n[4] CURATED PLAYLISTS") + print("-" * 40) + + try: + with db._get_connection() as conn: + cursor = conn.cursor() + + playlists_to_check = [ + 'release_radar', + 'release_radar_spotify', + 'release_radar_itunes', + 'discovery_weekly', + 'discovery_weekly_spotify', + 'discovery_weekly_itunes' + ] + + for playlist_type in playlists_to_check: + cursor.execute(""" + SELECT track_ids_json FROM discovery_curated_playlists + WHERE playlist_type = ? + """, (playlist_type,)) + row = cursor.fetchone() + + if row: + track_ids = json.loads(row['track_ids_json']) + status = f"{len(track_ids)} tracks" + if len(track_ids) == 0: + status += " [EMPTY]" + else: + status = "[NOT FOUND]" + + print(f" {playlist_type}: {status}") + + # Check iTunes-specific playlists + cursor.execute(""" + SELECT track_ids_json FROM discovery_curated_playlists + WHERE playlist_type = 'release_radar_itunes' + """) + itunes_rr = cursor.fetchone() + + cursor.execute(""" + SELECT track_ids_json FROM discovery_curated_playlists + WHERE playlist_type = 'discovery_weekly_itunes' + """) + itunes_dw = cursor.fetchone() + + if not itunes_rr or len(json.loads(itunes_rr['track_ids_json'])) == 0: + print("\n [CRITICAL] release_radar_itunes is empty or missing!") + if not itunes_dw or len(json.loads(itunes_dw['track_ids_json'])) == 0: + print(" [CRITICAL] discovery_weekly_itunes is empty or missing!") + + except Exception as e: + print(f" [ERROR] Could not check curated playlists: {e}") + + # 5. Check Watchlist Artists + print("\n[5] WATCHLIST ARTISTS") + print("-" * 40) + + try: + with db._get_connection() as conn: + cursor = conn.cursor() + + # Total artists + cursor.execute("SELECT COUNT(*) as total FROM watchlist_artists") + total = cursor.fetchone()['total'] + + # With iTunes IDs + cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE itunes_artist_id IS NOT NULL") + with_itunes = cursor.fetchone()['count'] + + # With Spotify IDs + cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE spotify_artist_id IS NOT NULL") + with_spotify = cursor.fetchone()['count'] + + print(f" Total watchlist artists: {total}") + print(f" With iTunes ID: {with_itunes} ({100*with_itunes/total:.1f}%)" if total > 0 else " With iTunes ID: 0") + print(f" With Spotify ID: {with_spotify} ({100*with_spotify/total:.1f}%)" if total > 0 else " With Spotify ID: 0") + + if with_itunes == 0 and total > 0: + print(" [WARNING] No watchlist artists have iTunes IDs - source artist data limited") + + except Exception as e: + print(f" [ERROR] Could not check watchlist artists: {e}") + + # Summary + print("\n" + "=" * 60) + print("SUMMARY & RECOMMENDED ACTIONS") + print("=" * 60) + print(""" +If you see [CRITICAL] or [WARNING] messages above, follow these steps: + +QUICK FIX - Force Refresh Discover Data: +----------------------------------------- +Call the API endpoint to refresh discover data: + curl -X POST http://localhost:5000/api/discover/refresh + +This will: +- Cache recent albums from your watchlist artists +- Create curated playlists (Release Radar & Discovery Weekly) + +FULL FIX - Run Watchlist Scan: +------------------------------ +1. Go to the web UI Settings page +2. Click "Scan Watchlist" button +3. Wait for scan to complete + +This will: +- Fetch similar artists from MusicMap for each watchlist artist +- Populate the discovery pool with tracks +- Cache recent albums +- Create curated playlists + +ROOT CAUSE NOTES: +----------------- +- Similar artists = 0: MusicMap fetch may have failed. Watchlist scan needed. +- Recent albums = 0: cache_discovery_recent_albums() needs to run. +- Curated playlists missing: curate_discovery_playlists() needs to run. + +The discover page will now fall back to watchlist artists if similar +artists are not available, so basic functionality should still work. +""") + + +if __name__ == '__main__': + diagnose_itunes_discover() diff --git a/web_server.py b/web_server.py index 350abaad..0476b16b 100644 --- a/web_server.py +++ b/web_server.py @@ -14327,6 +14327,105 @@ def search_itunes_tracks(): return jsonify({"error": str(e)}), 500 +@app.route('/api/itunes/album/', methods=['GET']) +def get_itunes_album_tracks(album_id): + """Fetches full track details for a specific iTunes album.""" + try: + from core.itunes_client import iTunesClient + + itunes_client = iTunesClient() + album_data = itunes_client.get_album(album_id) + + if not album_data: + return jsonify({"error": "Album not found"}), 404 + + # Get tracks for this album + tracks_data = itunes_client.get_album_tracks(album_id) + tracks = tracks_data.get('items', []) if tracks_data else [] + + # Format response to match Spotify structure for frontend compatibility + album_dict = { + 'id': album_data.get('id', album_id), + 'name': album_data.get('name', 'Unknown Album'), + 'artists': album_data.get('artists', []), + 'release_date': album_data.get('release_date', ''), + 'total_tracks': album_data.get('total_tracks', len(tracks)), + 'album_type': album_data.get('album_type', 'album'), + 'images': album_data.get('images', []), + 'tracks': tracks, + 'source': 'itunes' + } + return jsonify(album_dict) + + except Exception as e: + logger.error(f"Error fetching iTunes album tracks: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route('/api/discover/album//', methods=['GET']) +def get_discover_album(source, album_id): + """ + Source-agnostic album endpoint for discover page. + Fetches album from the appropriate source (spotify or itunes). + """ + try: + if source == 'spotify': + if not spotify_client or not spotify_client.is_authenticated(): + return jsonify({"error": "Spotify not authenticated."}), 401 + + album_data = spotify_client.get_album(album_id) + if not album_data: + return jsonify({"error": "Album not found"}), 404 + + tracks = album_data.get('tracks', {}).get('items', []) + if not tracks: + tracks_data = spotify_client.get_album_tracks(album_id) + if tracks_data and 'items' in tracks_data: + tracks = tracks_data['items'] + + return jsonify({ + 'id': album_data['id'], + 'name': album_data['name'], + 'artists': album_data.get('artists', []), + 'release_date': album_data.get('release_date', ''), + 'total_tracks': album_data.get('total_tracks', 0), + 'album_type': album_data.get('album_type', 'album'), + 'images': album_data.get('images', []), + 'tracks': tracks, + 'source': 'spotify' + }) + + elif source == 'itunes': + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() + + album_data = itunes_client.get_album(album_id) + if not album_data: + return jsonify({"error": "Album not found"}), 404 + + tracks_data = itunes_client.get_album_tracks(album_id) + tracks = tracks_data.get('items', []) if tracks_data else [] + + return jsonify({ + 'id': album_data.get('id', album_id), + 'name': album_data.get('name', 'Unknown Album'), + 'artists': album_data.get('artists', []), + 'release_date': album_data.get('release_date', ''), + 'total_tracks': album_data.get('total_tracks', len(tracks)), + 'album_type': album_data.get('album_type', 'album'), + 'images': album_data.get('images', []), + 'tracks': tracks, + 'source': 'itunes' + }) + + else: + return jsonify({"error": f"Unknown source: {source}"}), 400 + + except Exception as e: + logger.error(f"Error fetching discover album: {e}") + return jsonify({"error": str(e)}), 500 + + # =================================================================== # TIDAL PLAYLIST API ENDPOINTS # =================================================================== @@ -17618,7 +17717,24 @@ def start_watchlist_scan(): })()) print(f"✅ Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist") - + + # Fetch similar artists for discovery feature + # This is critical for the discover page to work + try: + watchlist_scan_state['current_phase'] = 'fetching_similar_artists' + source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id) + + if database.has_fresh_similar_artists(source_artist_id, days_threshold=30): + print(f" Similar artists for {artist.artist_name} are cached and fresh") + # Still backfill missing iTunes IDs + scanner._backfill_similar_artists_itunes_ids(source_artist_id) + else: + print(f" Fetching similar artists for {artist.artist_name}...") + scanner.update_similar_artists(artist) + print(f" Similar artists updated for {artist.artist_name}") + except Exception as similar_error: + print(f" ⚠️ Failed to update similar artists for {artist.artist_name}: {similar_error}") + # Delay between artists if i < len(watchlist_artists) - 1: watchlist_scan_state['current_phase'] = 'rate_limiting' @@ -18469,11 +18585,63 @@ def get_discover_hero(): active_source = _get_active_discovery_source() print(f"🎵 Discover hero using source: {active_source}") + # Import iTunes client for fallback + from core.itunes_client import iTunesClient + itunes_client = iTunesClient() + # Get top similar artists (by occurrence count) - get 20 for variety similar_artists = database.get_top_similar_artists(limit=20) + # FALLBACK: If no similar artists exist, use watchlist artists for Hero section if not similar_artists: - return jsonify({"success": True, "artists": [], "source": active_source}) + print("[Discover Hero] No similar artists found, falling back to watchlist artists") + watchlist_artists = database.get_watchlist_artists() + + if not watchlist_artists: + return jsonify({"success": True, "artists": [], "source": active_source}) + + # Convert watchlist artists to hero format + import random + shuffled_watchlist = list(watchlist_artists) + random.shuffle(shuffled_watchlist) + + hero_artists = [] + for artist in shuffled_watchlist[:10]: + artist_id = artist.itunes_artist_id if active_source == 'itunes' else artist.spotify_artist_id + if not artist_id: + continue + + artist_data = { + "spotify_artist_id": artist.spotify_artist_id, + "itunes_artist_id": artist.itunes_artist_id, + "artist_id": artist_id, + "artist_name": artist.artist_name, + "occurrence_count": 1, + "similarity_rank": 1, + "source": active_source, + "is_watchlist": True + } + + # Try to get artist image + try: + if active_source == 'itunes' and artist.itunes_artist_id: + itunes_artist = itunes_client.get_artist(artist.itunes_artist_id) + if itunes_artist: + artist_data['image_url'] = itunes_artist.get('images', [{}])[0].get('url') if itunes_artist.get('images') else None + artist_data['genres'] = itunes_artist.get('genres', []) + elif active_source == 'spotify' and artist.spotify_artist_id: + if spotify_client and spotify_client.is_authenticated(): + sp_artist = spotify_client.get_artist(artist.spotify_artist_id) + if sp_artist and sp_artist.get('images'): + artist_data['image_url'] = sp_artist['images'][0]['url'] if sp_artist['images'] else None + artist_data['genres'] = sp_artist.get('genres', []) + except Exception as img_err: + print(f"Could not fetch watchlist artist image: {img_err}") + + hero_artists.append(artist_data) + + print(f"[Discover Hero] Returning {len(hero_artists)} watchlist artists as fallback") + return jsonify({"success": True, "artists": hero_artists, "source": active_source, "fallback": "watchlist"}) # Filter to artists that have the appropriate ID for the active source valid_artists = [] @@ -18486,16 +18654,41 @@ def get_discover_hero(): elif artist.similar_artist_spotify_id and artist.similar_artist_itunes_id: valid_artists.append(artist) + # FALLBACK: If no valid artists for iTunes, try to resolve iTunes IDs on-the-fly + if active_source == 'itunes' and not valid_artists: + print(f"[iTunes Fallback] No artists with iTunes IDs found, attempting on-the-fly resolution for {len(similar_artists)} artists") + resolved_count = 0 + for artist in similar_artists: + if artist.similar_artist_itunes_id: + valid_artists.append(artist) + continue + # Try to resolve iTunes ID by name + try: + itunes_results = itunes_client.search_artists(artist.similar_artist_name, limit=1) + if itunes_results and len(itunes_results) > 0: + itunes_id = itunes_results[0].id + # Cache the resolved ID for future use + database.update_similar_artist_itunes_id(artist.id, itunes_id) + # Create a modified artist object with the resolved ID + artist.similar_artist_itunes_id = itunes_id + valid_artists.append(artist) + resolved_count += 1 + print(f" [Resolved] {artist.similar_artist_name} -> iTunes ID: {itunes_id}") + except Exception as resolve_err: + print(f" [Failed] Could not resolve iTunes ID for {artist.similar_artist_name}: {resolve_err}") + # Stop after 10 successful resolutions to avoid rate limiting + if len(valid_artists) >= 10: + break + print(f"[iTunes Fallback] Resolved {resolved_count} artists with iTunes IDs") + + print(f"[Discover Hero] Found {len(valid_artists)} valid artists for source: {active_source}") + # Shuffle for variety and take top 10 import random shuffled = list(valid_artists) random.shuffle(shuffled) similar_artists = shuffled[:10] - # Import iTunes client for fallback - from core.itunes_client import iTunesClient - itunes_client = iTunesClient() - # Convert to JSON format with data enrichment from appropriate source hero_artists = [] for artist in similar_artists: @@ -18685,6 +18878,121 @@ def get_discover_weekly(): print(f"Error getting discovery weekly: {e}") return jsonify({"success": False, "error": str(e)}), 500 + +@app.route('/api/discover/refresh', methods=['POST']) +def refresh_discover_data(): + """ + Force refresh discover page data (recent albums cache and curated playlists). + Useful for initial setup or when data appears stale. + """ + try: + from core.watchlist_scanner import WatchlistScanner + + database = get_database() + scanner = WatchlistScanner(spotify_client, database) + + print("[Discover Refresh] Starting forced refresh of discover data...") + + # Cache recent albums from watchlist and similar artists + print("[Discover Refresh] Caching recent albums...") + scanner.cache_discovery_recent_albums() + + # Curate playlists + print("[Discover Refresh] Curating discovery playlists...") + scanner.curate_discovery_playlists() + + # Get counts for response + active_source = _get_active_discovery_source() + recent_albums = database.get_discovery_recent_albums(limit=100, source=active_source) + release_radar = database.get_curated_playlist(f'release_radar_{active_source}') or [] + discovery_weekly = database.get_curated_playlist(f'discovery_weekly_{active_source}') or [] + + print(f"[Discover Refresh] Complete! Recent albums: {len(recent_albums)}, Release Radar: {len(release_radar)} tracks, Discovery Weekly: {len(discovery_weekly)} tracks") + + return jsonify({ + "success": True, + "message": "Discover data refreshed", + "source": active_source, + "recent_albums_count": len(recent_albums), + "release_radar_tracks": len(release_radar), + "discovery_weekly_tracks": len(discovery_weekly) + }) + + except Exception as e: + print(f"Error refreshing discover data: {e}") + import traceback + traceback.print_exc() + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/discover/diagnose', methods=['GET']) +def diagnose_discover_data(): + """ + Diagnostic endpoint to check the state of discover data. + Returns counts of similar artists, discovery pool, recent albums, etc. + """ + try: + database = get_database() + active_source = _get_active_discovery_source() + + with database._get_connection() as conn: + cursor = conn.cursor() + + # Similar artists stats + cursor.execute("SELECT COUNT(*) as total FROM similar_artists") + total_similar = cursor.fetchone()['total'] + + cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_itunes_id IS NOT NULL") + similar_with_itunes = cursor.fetchone()['count'] + + cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_spotify_id IS NOT NULL") + similar_with_spotify = cursor.fetchone()['count'] + + # Discovery pool stats + cursor.execute("SELECT source, COUNT(*) as count FROM discovery_pool GROUP BY source") + pool_by_source = {row['source']: row['count'] for row in cursor.fetchall()} + + # Recent albums stats + cursor.execute("SELECT source, COUNT(*) as count FROM discovery_recent_albums GROUP BY source") + albums_by_source = {row['source']: row['count'] for row in cursor.fetchall()} + + # Curated playlists + cursor.execute("SELECT playlist_type, track_ids_json FROM discovery_curated_playlists") + playlists = {} + for row in cursor.fetchall(): + import json + track_ids = json.loads(row['track_ids_json']) if row['track_ids_json'] else [] + playlists[row['playlist_type']] = len(track_ids) + + # Watchlist artists + cursor.execute("SELECT COUNT(*) as total FROM watchlist_artists") + total_watchlist = cursor.fetchone()['total'] + + cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE itunes_artist_id IS NOT NULL") + watchlist_with_itunes = cursor.fetchone()['count'] + + return jsonify({ + "success": True, + "active_source": active_source, + "similar_artists": { + "total": total_similar, + "with_itunes_id": similar_with_itunes, + "with_spotify_id": similar_with_spotify + }, + "discovery_pool": pool_by_source, + "recent_albums": albums_by_source, + "curated_playlists": playlists, + "watchlist_artists": { + "total": total_watchlist, + "with_itunes_id": watchlist_with_itunes + } + }) + + except Exception as e: + print(f"Error diagnosing discover data: {e}") + return jsonify({"success": False, "error": str(e)}), 500 + + # ======================================== # SEASONAL DISCOVERY ENDPOINTS # ======================================== diff --git a/webui/static/script.js b/webui/static/script.js index 98cd6e44..8e6e1a2b 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -33121,8 +33121,16 @@ async function openDownloadModalForRecentAlbum(albumIndex) { showLoadingOverlay(`Loading tracks for ${album.album_name}...`); try { - // Fetch album tracks from Spotify API via backend - const response = await fetch(`/api/spotify/album/${album.album_spotify_id}`); + // Determine source and album ID - use source-agnostic endpoint + const source = album.source || (album.album_spotify_id ? 'spotify' : 'itunes'); + const albumId = source === 'spotify' ? album.album_spotify_id : album.album_itunes_id; + + if (!albumId) { + throw new Error(`No ${source} album ID available`); + } + + // Fetch album tracks from appropriate source via backend + const response = await fetch(`/api/discover/album/${source}/${albumId}`); if (!response.ok) { throw new Error('Failed to fetch album tracks'); } @@ -33157,13 +33165,14 @@ async function openDownloadModalForRecentAlbum(albumIndex) { }; }); - // Create virtual playlist ID - const virtualPlaylistId = `discover_album_${album.album_spotify_id}`; + // Create virtual playlist ID using the appropriate album ID + const virtualPlaylistId = `discover_album_${albumId}`; // CRITICAL FIX: Pass proper artist/album context for modal display const artistContext = { - id: album.artist_spotify_id, - name: album.artist_name + id: source === 'spotify' ? album.artist_spotify_id : album.artist_itunes_id, + name: album.artist_name, + source: source }; const albumContext = {