Fix iTunes-only Discover page not loading data

- Add similar artists fetching to web UI scan loop
  - Add database migration for UNIQUE constraint on similar_artists table                                          - Add source-agnostic /api/discover/album endpoint for iTunes support
  - Fix NOT NULL constraint on discovery_recent_albums blocking iTunes albums
  - Add fallback to watchlist artists when no similar artists exist
  - Add /api/discover/refresh and /api/discover/diagnose endpoints
  - Add retry logic with exponential backoff for iTunes API calls
  - Ensure cache_discovery_recent_albums runs even when pool population skips

core/watchlist_scanner.py

@@ -21,9 +21,61 @@ logger = get_logger("watchlist_scanner")
 
 # Rate limiting constants for watchlist operations
 DELAY_BETWEEN_ARTISTS = 2.0      # 2 seconds between different artists
-DELAY_BETWEEN_ALBUMS = 0.5       # 500ms between albums for same artist  
+DELAY_BETWEEN_ALBUMS = 0.5       # 500ms between albums for same artist
 DELAY_BETWEEN_API_BATCHES = 1.0  # 1 second between API batch operations
 
+# iTunes API retry configuration
+ITUNES_MAX_RETRIES = 3
+ITUNES_BASE_DELAY = 1.0  # Base delay in seconds for exponential backoff
+
+
+def itunes_api_call_with_retry(func, *args, max_retries=ITUNES_MAX_RETRIES, **kwargs):
+    """
+    Execute an iTunes API call with exponential backoff retry logic.
+
+    Args:
+        func: The function to call
+        *args: Arguments to pass to the function
+        max_retries: Maximum number of retry attempts
+        **kwargs: Keyword arguments to pass to the function
+
+    Returns:
+        The result of the function call, or None if all retries failed
+    """
+    last_error = None
+    for attempt in range(max_retries):
+        try:
+            result = func(*args, **kwargs)
+            return result
+        except requests.exceptions.HTTPError as e:
+            # Handle rate limiting (429) and server errors (5xx)
+            if e.response is not None and e.response.status_code == 429:
+                delay = ITUNES_BASE_DELAY * (2 ** attempt)
+                logger.warning(f"[iTunes] Rate limited, retrying in {delay}s (attempt {attempt + 1}/{max_retries})")
+                time.sleep(delay)
+                last_error = e
+            elif e.response is not None and e.response.status_code >= 500:
+                delay = ITUNES_BASE_DELAY * (2 ** attempt)
+                logger.warning(f"[iTunes] Server error {e.response.status_code}, retrying in {delay}s (attempt {attempt + 1}/{max_retries})")
+                time.sleep(delay)
+                last_error = e
+            else:
+                raise  # Don't retry on client errors (4xx except 429)
+        except requests.exceptions.RequestException as e:
+            # Retry on connection errors
+            delay = ITUNES_BASE_DELAY * (2 ** attempt)
+            logger.warning(f"[iTunes] Connection error, retrying in {delay}s (attempt {attempt + 1}/{max_retries}): {e}")
+            time.sleep(delay)
+            last_error = e
+        except Exception as e:
+            # Don't retry on other exceptions
+            raise
+
+    if last_error:
+        logger.error(f"[iTunes] All {max_retries} retry attempts failed: {last_error}")
+    return None
+
+
 def clean_track_name_for_search(track_name):
     """
     Intelligently cleans a track name for searching by removing noise while preserving important version information.
@@ -1287,9 +1339,11 @@ class WatchlistScanner:
                         except Exception as e:
                             logger.debug(f"Spotify match failed for {artist_name_to_match}: {e}")
 
-                    # Try to match on iTunes
+                    # Try to match on iTunes (with retry for rate limiting)
                     try:
-                        itunes_results = itunes_client.search_artists(artist_name_to_match, limit=1)
+                        itunes_results = itunes_api_call_with_retry(
+                            itunes_client.search_artists, artist_name_to_match, limit=1
+                        )
                         if itunes_results and len(itunes_results) > 0:
                             itunes_artist = itunes_results[0]
                             # Skip if this is the searched artist
@@ -1301,8 +1355,10 @@ class WatchlistScanner:
                                 # Use iTunes genres if we don't have Spotify genres
                                 if not artist_data['genres'] and hasattr(itunes_artist, 'genres'):
                                     artist_data['genres'] = itunes_artist.genres
+                        else:
+                            logger.info(f"  [iTunes] No match found for: {artist_name_to_match}")
                     except Exception as e:
-                        logger.debug(f"iTunes match failed for {artist_name_to_match}: {e}")
+                        logger.info(f"  [iTunes] Match failed for {artist_name_to_match}: {e}")
 
                     # Only add if we got at least one ID
                     if artist_data['spotify_id'] or artist_data['itunes_id']:
@@ -1314,7 +1370,11 @@ class WatchlistScanner:
                     logger.debug(f"Error matching {artist_name_to_match}: {match_error}")
                     continue
 
-            logger.info(f"Matched {len(matched_artists)} similar artists (Spotify + iTunes)")
+            # Log detailed matching statistics
+            itunes_matched = sum(1 for a in matched_artists if a.get('itunes_id'))
+            spotify_matched = sum(1 for a in matched_artists if a.get('spotify_id'))
+            both_matched = sum(1 for a in matched_artists if a.get('itunes_id') and a.get('spotify_id'))
+            logger.info(f"Matched {len(matched_artists)} similar artists - iTunes: {itunes_matched}, Spotify: {spotify_matched}, Both: {both_matched}")
             return matched_artists
 
         except requests.exceptions.RequestException as e:
@@ -1436,9 +1496,15 @@ class WatchlistScanner:
             from datetime import datetime, timedelta
             import random
 
-            # Check if we should run (prevents over-polling)
+            # Check if we should run discovery pool population (prevents over-polling)
-            if not self.database.should_populate_discovery_pool(hours_threshold=24):
+            skip_pool_population = not self.database.should_populate_discovery_pool(hours_threshold=24)
-                logger.info("Discovery pool was populated recently (< 24 hours ago). Skipping.")
+
+            if skip_pool_population:
+                logger.info("Discovery pool was populated recently (< 24 hours ago). Skipping pool population.")
+                logger.info("But still refreshing recent albums cache and curated playlists...")
+                # Still run these even when skipping main pool population
+                self.cache_discovery_recent_albums()
+                self.curate_discovery_playlists()
                 return
 
             logger.info("Populating discovery pool from similar artists...")
@@ -1461,7 +1527,11 @@ class WatchlistScanner:
             similar_artists = self.database.get_top_similar_artists(limit=top_artists_limit)
 
             if not similar_artists:
-                logger.info("No similar artists found to populate discovery pool")
+                logger.info("No similar artists found to populate discovery pool from similar artists")
+                logger.info("But still caching recent albums from watchlist artists and curating playlists...")
+                # Still run these even without similar artists - they use watchlist artists
+                self.cache_discovery_recent_albums()
+                self.curate_discovery_playlists()
                 return
 
             logger.info(f"Processing {len(similar_artists)} top similar artists for discovery pool")
@@ -2056,26 +2126,40 @@ class WatchlistScanner:
                     logger.debug(f"Error processing album: {e}")
                 return False
 
+            # Track resolution stats
+            itunes_resolved = 0
+            itunes_failed_resolve = 0
+
             # Process watchlist artists
             for artist in watchlist_artists:
                 # Always process iTunes (baseline)
                 itunes_id = artist.itunes_artist_id
                 if not itunes_id:
-                    # Try to resolve iTunes ID on-the-fly
+                    # Try to resolve iTunes ID on-the-fly (with retry for rate limiting)
                     try:
-                        results = itunes_client.search_artists(artist.artist_name, limit=1)
+                        results = itunes_api_call_with_retry(
+                            itunes_client.search_artists, artist.artist_name, limit=1
+                        )
                         if results and len(results) > 0:
                             itunes_id = results[0].id
-                    except:
+                            itunes_resolved += 1
-                        pass
+                            logger.debug(f"[iTunes] Resolved ID for {artist.artist_name}: {itunes_id}")
+                        else:
+                            itunes_failed_resolve += 1
+                            logger.info(f"[iTunes] No artist found for: {artist.artist_name}")
+                    except Exception as e:
+                        itunes_failed_resolve += 1
+                        logger.info(f"[iTunes] Failed to resolve {artist.artist_name}: {e}")
 
                 if itunes_id:
                     try:
-                        albums = itunes_client.get_artist_albums(itunes_id, album_type='album,single', limit=20)
+                        albums = itunes_api_call_with_retry(
+                            itunes_client.get_artist_albums, itunes_id, album_type='album,single', limit=20
+                        )
                         for album in albums or []:
                             process_album(album, artist.artist_name, artist.spotify_artist_id, itunes_id, 'itunes')
                     except Exception as e:
-                        logger.debug(f"Error fetching iTunes albums for {artist.artist_name}: {e}")
+                        logger.info(f"[iTunes] Error fetching albums for {artist.artist_name}: {e}")
 
                 # Process Spotify if authenticated
                 if spotify_available and artist.spotify_artist_id:
@@ -2097,23 +2181,33 @@ class WatchlistScanner:
                 # Always process iTunes (baseline)
                 itunes_id = artist.similar_artist_itunes_id
                 if not itunes_id:
-                    # Try to resolve iTunes ID on-the-fly
+                    # Try to resolve iTunes ID on-the-fly (with retry for rate limiting)
                     try:
-                        results = itunes_client.search_artists(artist.similar_artist_name, limit=1)
+                        results = itunes_api_call_with_retry(
+                            itunes_client.search_artists, artist.similar_artist_name, limit=1
+                        )
                         if results and len(results) > 0:
                             itunes_id = results[0].id
                             # Cache for future
                             self.database.update_similar_artist_itunes_id(artist.id, itunes_id)
-                    except:
+                            itunes_resolved += 1
-                        pass
+                            logger.debug(f"[iTunes] Resolved ID for similar artist {artist.similar_artist_name}: {itunes_id}")
+                        else:
+                            itunes_failed_resolve += 1
+                            logger.info(f"[iTunes] No artist found for similar: {artist.similar_artist_name}")
+                    except Exception as e:
+                        itunes_failed_resolve += 1
+                        logger.info(f"[iTunes] Failed to resolve similar {artist.similar_artist_name}: {e}")
 
                 if itunes_id:
                     try:
-                        albums = itunes_client.get_artist_albums(itunes_id, album_type='album,single', limit=20)
+                        albums = itunes_api_call_with_retry(
+                            itunes_client.get_artist_albums, itunes_id, album_type='album,single', limit=20
+                        )
                         for album in albums or []:
                             process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, itunes_id, 'itunes')
                     except Exception as e:
-                        logger.debug(f"Error fetching iTunes albums for {artist.similar_artist_name}: {e}")
+                        logger.info(f"[iTunes] Error fetching albums for similar {artist.similar_artist_name}: {e}")
 
                 # Process Spotify if authenticated
                 if spotify_available and artist.similar_artist_spotify_id:
@@ -2132,6 +2226,7 @@ class WatchlistScanner:
 
             total_cached = cached_count['spotify'] + cached_count['itunes']
             logger.info(f"Cached {total_cached} recent albums (Spotify: {cached_count['spotify']}, iTunes: {cached_count['itunes']}) from {albums_checked} albums checked")
+            logger.info(f"[iTunes] ID resolution stats: {itunes_resolved} resolved, {itunes_failed_resolve} failed")
 
         except Exception as e:
             logger.error(f"Error caching discovery recent albums: {e}")
@@ -2171,6 +2266,9 @@ class WatchlistScanner:
                 recent_albums = self.database.get_discovery_recent_albums(limit=50, source=source)
                 release_radar_tracks = []
 
+                if not recent_albums:
+                    logger.warning(f"[{source.upper()}] No recent albums found for Release Radar - check cache_discovery_recent_albums()")
+
                 if recent_albums:
                     # Group albums by artist for variety
                     albums_by_artist = {}
@@ -2196,7 +2294,9 @@ class WatchlistScanner:
                                 if source == 'spotify':
                                     album_data = self.spotify_client.get_album(album_id)
                                 else:
-                                    album_data = itunes_client.get_album(album_id)
+                                    album_data = itunes_api_call_with_retry(
+                                        itunes_client.get_album, album_id
+                                    )
 
                                 if not album_data or 'tracks' not in album_data:
                                     continue
@@ -2295,6 +2395,9 @@ class WatchlistScanner:
                 logger.info(f"Curating Discovery Weekly for {source}...")
                 discovery_tracks = self.database.get_discovery_pool_tracks(limit=2000, new_releases_only=False, source=source)
 
+                if not discovery_tracks:
+                    logger.warning(f"[{source.upper()}] No discovery pool tracks found for Discovery Weekly - check populate_discovery_pool()")
+
                 discovery_weekly_tracks = []
                 if discovery_tracks:
                     # Separate tracks by popularity tiers

database/music_database.py

@@ -636,6 +636,90 @@ class MusicDatabase:
                 cursor.execute("ALTER TABLE discovery_recent_albums ADD COLUMN source TEXT DEFAULT 'spotify'")
                 logger.info("Added iTunes columns to discovery_recent_albums table for dual-source discovery")
 
+            # Migration: Fix NOT NULL constraint on album_spotify_id (required for iTunes-only albums)
+            # Check if album_spotify_id has NOT NULL constraint by checking table schema
+            cursor.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='discovery_recent_albums'")
+            table_schema = cursor.fetchone()
+            if table_schema and 'album_spotify_id TEXT NOT NULL' in (table_schema[0] or ''):
+                logger.info("Migrating discovery_recent_albums to allow NULL album_spotify_id for iTunes support...")
+                # SQLite doesn't support ALTER COLUMN, so recreate table
+                cursor.execute("""
+                    CREATE TABLE IF NOT EXISTS discovery_recent_albums_new (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        album_spotify_id TEXT,
+                        album_itunes_id TEXT,
+                        artist_spotify_id TEXT,
+                        artist_itunes_id TEXT,
+                        source TEXT NOT NULL DEFAULT 'spotify',
+                        album_name TEXT NOT NULL,
+                        artist_name TEXT NOT NULL,
+                        album_cover_url TEXT,
+                        release_date TEXT NOT NULL,
+                        album_type TEXT DEFAULT 'album',
+                        cached_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                        UNIQUE(album_spotify_id, album_itunes_id, source)
+                    )
+                """)
+                cursor.execute("""
+                    INSERT OR IGNORE INTO discovery_recent_albums_new
+                    SELECT * FROM discovery_recent_albums
+                """)
+                cursor.execute("DROP TABLE discovery_recent_albums")
+                cursor.execute("ALTER TABLE discovery_recent_albums_new RENAME TO discovery_recent_albums")
+                conn.commit()
+                logger.info("Successfully migrated discovery_recent_albums table for iTunes support")
+
+            # Migration: Add UNIQUE constraint to similar_artists table
+            # Test if ON CONFLICT works by trying a dummy operation
+            needs_similar_migration = False
+            try:
+                cursor.execute("""
+                    INSERT INTO similar_artists
+                    (source_artist_id, similar_artist_name, similarity_rank, occurrence_count, last_updated)
+                    VALUES ('__migration_test__', '__migration_test__', 1, 1, CURRENT_TIMESTAMP)
+                    ON CONFLICT(source_artist_id, similar_artist_name)
+                    DO UPDATE SET occurrence_count = occurrence_count
+                """)
+                # Clean up test row
+                cursor.execute("DELETE FROM similar_artists WHERE source_artist_id = '__migration_test__'")
+                logger.info("similar_artists table has correct UNIQUE constraint")
+            except Exception as constraint_error:
+                logger.info(f"similar_artists needs migration (constraint test failed: {constraint_error})")
+                needs_similar_migration = True
+
+            if needs_similar_migration:
+                logger.info("Migrating similar_artists to add UNIQUE constraint...")
+                # Get a fresh connection for the migration
+                with self._get_connection() as migration_conn:
+                    migration_cursor = migration_conn.cursor()
+                    # SQLite doesn't support adding constraints, so recreate table
+                    migration_cursor.execute("DROP TABLE IF EXISTS similar_artists_new")
+                    migration_cursor.execute("""
+                        CREATE TABLE similar_artists_new (
+                            id INTEGER PRIMARY KEY AUTOINCREMENT,
+                            source_artist_id TEXT NOT NULL,
+                            similar_artist_spotify_id TEXT,
+                            similar_artist_itunes_id TEXT,
+                            similar_artist_name TEXT NOT NULL,
+                            similarity_rank INTEGER DEFAULT 1,
+                            occurrence_count INTEGER DEFAULT 1,
+                            last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                            UNIQUE(source_artist_id, similar_artist_name)
+                        )
+                    """)
+                    migration_cursor.execute("""
+                        INSERT OR IGNORE INTO similar_artists_new
+                        (source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id,
+                         similar_artist_name, similarity_rank, occurrence_count, last_updated)
+                        SELECT source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id,
+                               similar_artist_name, similarity_rank, occurrence_count, last_updated
+                        FROM similar_artists
+                    """)
+                    migration_cursor.execute("DROP TABLE similar_artists")
+                    migration_cursor.execute("ALTER TABLE similar_artists_new RENAME TO similar_artists")
+                    migration_conn.commit()
+                    logger.info("Successfully migrated similar_artists table with UNIQUE constraint")
+
             # ============== INDEXES (after migrations to ensure columns exist) ==============
             cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_source ON similar_artists (source_artist_id)")
             cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_spotify ON similar_artists (similar_artist_spotify_id)")

tools/diagnose_itunes_discover.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+"""
+Diagnostic script to check iTunes data availability for the Discover page.
+
+Run this script to identify issues with iTunes data population:
+- Similar artists missing iTunes IDs
+- Discovery pool tracks by source
+- Recent albums by source
+- Curated playlists status
+
+Usage:
+    python tools/diagnose_itunes_discover.py
+"""
+
+import sys
+import os
+import json
+
+# Add parent directory to path for imports
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from database.music_database import MusicDatabase
+
+
+def diagnose_itunes_discover():
+    """Run diagnostic checks for iTunes discover data."""
+
+    print("=" * 60)
+    print("iTunes Discover Page Diagnostic Report")
+    print("=" * 60)
+
+    db = MusicDatabase()
+
+    # 1. Check Similar Artists
+    print("\n[1] SIMILAR ARTISTS")
+    print("-" * 40)
+
+    try:
+        with db._get_connection() as conn:
+            cursor = conn.cursor()
+
+            # Total similar artists
+            cursor.execute("SELECT COUNT(*) as total FROM similar_artists")
+            total = cursor.fetchone()['total']
+
+            # With iTunes IDs
+            cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_itunes_id IS NOT NULL")
+            with_itunes = cursor.fetchone()['count']
+
+            # With Spotify IDs
+            cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_spotify_id IS NOT NULL")
+            with_spotify = cursor.fetchone()['count']
+
+            # With both
+            cursor.execute("""
+                SELECT COUNT(*) as count FROM similar_artists
+                WHERE similar_artist_itunes_id IS NOT NULL
+                AND similar_artist_spotify_id IS NOT NULL
+            """)
+            with_both = cursor.fetchone()['count']
+
+            print(f"  Total similar artists: {total}")
+            print(f"  With iTunes ID: {with_itunes} ({100*with_itunes/total:.1f}%)" if total > 0 else "  With iTunes ID: 0")
+            print(f"  With Spotify ID: {with_spotify} ({100*with_spotify/total:.1f}%)" if total > 0 else "  With Spotify ID: 0")
+            print(f"  With BOTH IDs: {with_both} ({100*with_both/total:.1f}%)" if total > 0 else "  With BOTH IDs: 0")
+
+            if with_itunes == 0 and total > 0:
+                print("  [CRITICAL] No similar artists have iTunes IDs - Hero section will be empty!")
+            elif with_itunes < total * 0.5:
+                print("  [WARNING] Less than 50% of similar artists have iTunes IDs")
+            else:
+                print("  [OK] iTunes coverage is adequate")
+
+    except Exception as e:
+        print(f"  [ERROR] Could not check similar artists: {e}")
+
+    # 2. Check Discovery Pool
+    print("\n[2] DISCOVERY POOL")
+    print("-" * 40)
+
+    try:
+        with db._get_connection() as conn:
+            cursor = conn.cursor()
+
+            # Total tracks
+            cursor.execute("SELECT COUNT(*) as total FROM discovery_pool")
+            total = cursor.fetchone()['total']
+
+            # By source
+            cursor.execute("""
+                SELECT source, COUNT(*) as count
+                FROM discovery_pool
+                GROUP BY source
+            """)
+            source_counts = {row['source']: row['count'] for row in cursor.fetchall()}
+
+            print(f"  Total tracks: {total}")
+            print(f"  Spotify tracks: {source_counts.get('spotify', 0)}")
+            print(f"  iTunes tracks: {source_counts.get('itunes', 0)}")
+
+            if source_counts.get('itunes', 0) == 0 and total > 0:
+                print("  [CRITICAL] No iTunes tracks in discovery pool - Fresh Tape/Archives will be empty!")
+            elif source_counts.get('itunes', 0) < total * 0.3:
+                print("  [WARNING] Low iTunes track count in discovery pool")
+            else:
+                print("  [OK] iTunes tracks present")
+
+    except Exception as e:
+        print(f"  [ERROR] Could not check discovery pool: {e}")
+
+    # 3. Check Recent Albums
+    print("\n[3] RECENT ALBUMS CACHE")
+    print("-" * 40)
+
+    try:
+        with db._get_connection() as conn:
+            cursor = conn.cursor()
+
+            # Total albums
+            cursor.execute("SELECT COUNT(*) as total FROM discovery_recent_albums")
+            total = cursor.fetchone()['total']
+
+            # By source
+            cursor.execute("""
+                SELECT source, COUNT(*) as count
+                FROM discovery_recent_albums
+                GROUP BY source
+            """)
+            source_counts = {row['source']: row['count'] for row in cursor.fetchall()}
+
+            print(f"  Total recent albums: {total}")
+            print(f"  Spotify albums: {source_counts.get('spotify', 0)}")
+            print(f"  iTunes albums: {source_counts.get('itunes', 0)}")
+
+            if source_counts.get('itunes', 0) == 0 and total > 0:
+                print("  [CRITICAL] No iTunes albums cached - Recent Releases section will be empty!")
+            elif source_counts.get('itunes', 0) < 5:
+                print("  [WARNING] Very few iTunes albums cached")
+            else:
+                print("  [OK] iTunes albums cached")
+
+    except Exception as e:
+        print(f"  [ERROR] Could not check recent albums: {e}")
+
+    # 4. Check Curated Playlists
+    print("\n[4] CURATED PLAYLISTS")
+    print("-" * 40)
+
+    try:
+        with db._get_connection() as conn:
+            cursor = conn.cursor()
+
+            playlists_to_check = [
+                'release_radar',
+                'release_radar_spotify',
+                'release_radar_itunes',
+                'discovery_weekly',
+                'discovery_weekly_spotify',
+                'discovery_weekly_itunes'
+            ]
+
+            for playlist_type in playlists_to_check:
+                cursor.execute("""
+                    SELECT track_ids_json FROM discovery_curated_playlists
+                    WHERE playlist_type = ?
+                """, (playlist_type,))
+                row = cursor.fetchone()
+
+                if row:
+                    track_ids = json.loads(row['track_ids_json'])
+                    status = f"{len(track_ids)} tracks"
+                    if len(track_ids) == 0:
+                        status += " [EMPTY]"
+                else:
+                    status = "[NOT FOUND]"
+
+                print(f"  {playlist_type}: {status}")
+
+            # Check iTunes-specific playlists
+            cursor.execute("""
+                SELECT track_ids_json FROM discovery_curated_playlists
+                WHERE playlist_type = 'release_radar_itunes'
+            """)
+            itunes_rr = cursor.fetchone()
+
+            cursor.execute("""
+                SELECT track_ids_json FROM discovery_curated_playlists
+                WHERE playlist_type = 'discovery_weekly_itunes'
+            """)
+            itunes_dw = cursor.fetchone()
+
+            if not itunes_rr or len(json.loads(itunes_rr['track_ids_json'])) == 0:
+                print("\n  [CRITICAL] release_radar_itunes is empty or missing!")
+            if not itunes_dw or len(json.loads(itunes_dw['track_ids_json'])) == 0:
+                print("  [CRITICAL] discovery_weekly_itunes is empty or missing!")
+
+    except Exception as e:
+        print(f"  [ERROR] Could not check curated playlists: {e}")
+
+    # 5. Check Watchlist Artists
+    print("\n[5] WATCHLIST ARTISTS")
+    print("-" * 40)
+
+    try:
+        with db._get_connection() as conn:
+            cursor = conn.cursor()
+
+            # Total artists
+            cursor.execute("SELECT COUNT(*) as total FROM watchlist_artists")
+            total = cursor.fetchone()['total']
+
+            # With iTunes IDs
+            cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE itunes_artist_id IS NOT NULL")
+            with_itunes = cursor.fetchone()['count']
+
+            # With Spotify IDs
+            cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE spotify_artist_id IS NOT NULL")
+            with_spotify = cursor.fetchone()['count']
+
+            print(f"  Total watchlist artists: {total}")
+            print(f"  With iTunes ID: {with_itunes} ({100*with_itunes/total:.1f}%)" if total > 0 else "  With iTunes ID: 0")
+            print(f"  With Spotify ID: {with_spotify} ({100*with_spotify/total:.1f}%)" if total > 0 else "  With Spotify ID: 0")
+
+            if with_itunes == 0 and total > 0:
+                print("  [WARNING] No watchlist artists have iTunes IDs - source artist data limited")
+
+    except Exception as e:
+        print(f"  [ERROR] Could not check watchlist artists: {e}")
+
+    # Summary
+    print("\n" + "=" * 60)
+    print("SUMMARY & RECOMMENDED ACTIONS")
+    print("=" * 60)
+    print("""
+If you see [CRITICAL] or [WARNING] messages above, follow these steps:
+
+QUICK FIX - Force Refresh Discover Data:
+-----------------------------------------
+Call the API endpoint to refresh discover data:
+  curl -X POST http://localhost:5000/api/discover/refresh
+
+This will:
+- Cache recent albums from your watchlist artists
+- Create curated playlists (Release Radar & Discovery Weekly)
+
+FULL FIX - Run Watchlist Scan:
+------------------------------
+1. Go to the web UI Settings page
+2. Click "Scan Watchlist" button
+3. Wait for scan to complete
+
+This will:
+- Fetch similar artists from MusicMap for each watchlist artist
+- Populate the discovery pool with tracks
+- Cache recent albums
+- Create curated playlists
+
+ROOT CAUSE NOTES:
+-----------------
+- Similar artists = 0: MusicMap fetch may have failed. Watchlist scan needed.
+- Recent albums = 0: cache_discovery_recent_albums() needs to run.
+- Curated playlists missing: curate_discovery_playlists() needs to run.
+
+The discover page will now fall back to watchlist artists if similar
+artists are not available, so basic functionality should still work.
+""")
+
+
+if __name__ == '__main__':
+    diagnose_itunes_discover()

web_server.py

@@ -14327,6 +14327,105 @@ def search_itunes_tracks():
         return jsonify({"error": str(e)}), 500
 
 
+@app.route('/api/itunes/album/<album_id>', methods=['GET'])
+def get_itunes_album_tracks(album_id):
+    """Fetches full track details for a specific iTunes album."""
+    try:
+        from core.itunes_client import iTunesClient
+
+        itunes_client = iTunesClient()
+        album_data = itunes_client.get_album(album_id)
+
+        if not album_data:
+            return jsonify({"error": "Album not found"}), 404
+
+        # Get tracks for this album
+        tracks_data = itunes_client.get_album_tracks(album_id)
+        tracks = tracks_data.get('items', []) if tracks_data else []
+
+        # Format response to match Spotify structure for frontend compatibility
+        album_dict = {
+            'id': album_data.get('id', album_id),
+            'name': album_data.get('name', 'Unknown Album'),
+            'artists': album_data.get('artists', []),
+            'release_date': album_data.get('release_date', ''),
+            'total_tracks': album_data.get('total_tracks', len(tracks)),
+            'album_type': album_data.get('album_type', 'album'),
+            'images': album_data.get('images', []),
+            'tracks': tracks,
+            'source': 'itunes'
+        }
+        return jsonify(album_dict)
+
+    except Exception as e:
+        logger.error(f"Error fetching iTunes album tracks: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
+@app.route('/api/discover/album/<source>/<album_id>', methods=['GET'])
+def get_discover_album(source, album_id):
+    """
+    Source-agnostic album endpoint for discover page.
+    Fetches album from the appropriate source (spotify or itunes).
+    """
+    try:
+        if source == 'spotify':
+            if not spotify_client or not spotify_client.is_authenticated():
+                return jsonify({"error": "Spotify not authenticated."}), 401
+
+            album_data = spotify_client.get_album(album_id)
+            if not album_data:
+                return jsonify({"error": "Album not found"}), 404
+
+            tracks = album_data.get('tracks', {}).get('items', [])
+            if not tracks:
+                tracks_data = spotify_client.get_album_tracks(album_id)
+                if tracks_data and 'items' in tracks_data:
+                    tracks = tracks_data['items']
+
+            return jsonify({
+                'id': album_data['id'],
+                'name': album_data['name'],
+                'artists': album_data.get('artists', []),
+                'release_date': album_data.get('release_date', ''),
+                'total_tracks': album_data.get('total_tracks', 0),
+                'album_type': album_data.get('album_type', 'album'),
+                'images': album_data.get('images', []),
+                'tracks': tracks,
+                'source': 'spotify'
+            })
+
+        elif source == 'itunes':
+            from core.itunes_client import iTunesClient
+            itunes_client = iTunesClient()
+
+            album_data = itunes_client.get_album(album_id)
+            if not album_data:
+                return jsonify({"error": "Album not found"}), 404
+
+            tracks_data = itunes_client.get_album_tracks(album_id)
+            tracks = tracks_data.get('items', []) if tracks_data else []
+
+            return jsonify({
+                'id': album_data.get('id', album_id),
+                'name': album_data.get('name', 'Unknown Album'),
+                'artists': album_data.get('artists', []),
+                'release_date': album_data.get('release_date', ''),
+                'total_tracks': album_data.get('total_tracks', len(tracks)),
+                'album_type': album_data.get('album_type', 'album'),
+                'images': album_data.get('images', []),
+                'tracks': tracks,
+                'source': 'itunes'
+            })
+
+        else:
+            return jsonify({"error": f"Unknown source: {source}"}), 400
+
+    except Exception as e:
+        logger.error(f"Error fetching discover album: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
 # ===================================================================
 # TIDAL PLAYLIST API ENDPOINTS
 # ===================================================================
@@ -17618,7 +17717,24 @@ def start_watchlist_scan():
                         })())
                         
                         print(f"✅ Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist")
-                        
+
+                        # Fetch similar artists for discovery feature
+                        # This is critical for the discover page to work
+                        try:
+                            watchlist_scan_state['current_phase'] = 'fetching_similar_artists'
+                            source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id)
+
+                            if database.has_fresh_similar_artists(source_artist_id, days_threshold=30):
+                                print(f"  Similar artists for {artist.artist_name} are cached and fresh")
+                                # Still backfill missing iTunes IDs
+                                scanner._backfill_similar_artists_itunes_ids(source_artist_id)
+                            else:
+                                print(f"  Fetching similar artists for {artist.artist_name}...")
+                                scanner.update_similar_artists(artist)
+                                print(f"  Similar artists updated for {artist.artist_name}")
+                        except Exception as similar_error:
+                            print(f"  ⚠️ Failed to update similar artists for {artist.artist_name}: {similar_error}")
+
                         # Delay between artists
                         if i < len(watchlist_artists) - 1:
                             watchlist_scan_state['current_phase'] = 'rate_limiting'
@@ -18469,11 +18585,63 @@ def get_discover_hero():
         active_source = _get_active_discovery_source()
         print(f"🎵 Discover hero using source: {active_source}")
 
+        # Import iTunes client for fallback
+        from core.itunes_client import iTunesClient
+        itunes_client = iTunesClient()
+
         # Get top similar artists (by occurrence count) - get 20 for variety
         similar_artists = database.get_top_similar_artists(limit=20)
 
+        # FALLBACK: If no similar artists exist, use watchlist artists for Hero section
         if not similar_artists:
-            return jsonify({"success": True, "artists": [], "source": active_source})
+            print("[Discover Hero] No similar artists found, falling back to watchlist artists")
+            watchlist_artists = database.get_watchlist_artists()
+
+            if not watchlist_artists:
+                return jsonify({"success": True, "artists": [], "source": active_source})
+
+            # Convert watchlist artists to hero format
+            import random
+            shuffled_watchlist = list(watchlist_artists)
+            random.shuffle(shuffled_watchlist)
+
+            hero_artists = []
+            for artist in shuffled_watchlist[:10]:
+                artist_id = artist.itunes_artist_id if active_source == 'itunes' else artist.spotify_artist_id
+                if not artist_id:
+                    continue
+
+                artist_data = {
+                    "spotify_artist_id": artist.spotify_artist_id,
+                    "itunes_artist_id": artist.itunes_artist_id,
+                    "artist_id": artist_id,
+                    "artist_name": artist.artist_name,
+                    "occurrence_count": 1,
+                    "similarity_rank": 1,
+                    "source": active_source,
+                    "is_watchlist": True
+                }
+
+                # Try to get artist image
+                try:
+                    if active_source == 'itunes' and artist.itunes_artist_id:
+                        itunes_artist = itunes_client.get_artist(artist.itunes_artist_id)
+                        if itunes_artist:
+                            artist_data['image_url'] = itunes_artist.get('images', [{}])[0].get('url') if itunes_artist.get('images') else None
+                            artist_data['genres'] = itunes_artist.get('genres', [])
+                    elif active_source == 'spotify' and artist.spotify_artist_id:
+                        if spotify_client and spotify_client.is_authenticated():
+                            sp_artist = spotify_client.get_artist(artist.spotify_artist_id)
+                            if sp_artist and sp_artist.get('images'):
+                                artist_data['image_url'] = sp_artist['images'][0]['url'] if sp_artist['images'] else None
+                                artist_data['genres'] = sp_artist.get('genres', [])
+                except Exception as img_err:
+                    print(f"Could not fetch watchlist artist image: {img_err}")
+
+                hero_artists.append(artist_data)
+
+            print(f"[Discover Hero] Returning {len(hero_artists)} watchlist artists as fallback")
+            return jsonify({"success": True, "artists": hero_artists, "source": active_source, "fallback": "watchlist"})
 
         # Filter to artists that have the appropriate ID for the active source
         valid_artists = []
@@ -18486,16 +18654,41 @@ def get_discover_hero():
             elif artist.similar_artist_spotify_id and artist.similar_artist_itunes_id:
                 valid_artists.append(artist)
 
+        # FALLBACK: If no valid artists for iTunes, try to resolve iTunes IDs on-the-fly
+        if active_source == 'itunes' and not valid_artists:
+            print(f"[iTunes Fallback] No artists with iTunes IDs found, attempting on-the-fly resolution for {len(similar_artists)} artists")
+            resolved_count = 0
+            for artist in similar_artists:
+                if artist.similar_artist_itunes_id:
+                    valid_artists.append(artist)
+                    continue
+                # Try to resolve iTunes ID by name
+                try:
+                    itunes_results = itunes_client.search_artists(artist.similar_artist_name, limit=1)
+                    if itunes_results and len(itunes_results) > 0:
+                        itunes_id = itunes_results[0].id
+                        # Cache the resolved ID for future use
+                        database.update_similar_artist_itunes_id(artist.id, itunes_id)
+                        # Create a modified artist object with the resolved ID
+                        artist.similar_artist_itunes_id = itunes_id
+                        valid_artists.append(artist)
+                        resolved_count += 1
+                        print(f"  [Resolved] {artist.similar_artist_name} -> iTunes ID: {itunes_id}")
+                except Exception as resolve_err:
+                    print(f"  [Failed] Could not resolve iTunes ID for {artist.similar_artist_name}: {resolve_err}")
+                # Stop after 10 successful resolutions to avoid rate limiting
+                if len(valid_artists) >= 10:
+                    break
+            print(f"[iTunes Fallback] Resolved {resolved_count} artists with iTunes IDs")
+
+        print(f"[Discover Hero] Found {len(valid_artists)} valid artists for source: {active_source}")
+
         # Shuffle for variety and take top 10
         import random
         shuffled = list(valid_artists)
         random.shuffle(shuffled)
         similar_artists = shuffled[:10]
 
-        # Import iTunes client for fallback
-        from core.itunes_client import iTunesClient
-        itunes_client = iTunesClient()
-
         # Convert to JSON format with data enrichment from appropriate source
         hero_artists = []
         for artist in similar_artists:
@@ -18685,6 +18878,121 @@ def get_discover_weekly():
         print(f"Error getting discovery weekly: {e}")
         return jsonify({"success": False, "error": str(e)}), 500
 
+
+@app.route('/api/discover/refresh', methods=['POST'])
+def refresh_discover_data():
+    """
+    Force refresh discover page data (recent albums cache and curated playlists).
+    Useful for initial setup or when data appears stale.
+    """
+    try:
+        from core.watchlist_scanner import WatchlistScanner
+
+        database = get_database()
+        scanner = WatchlistScanner(spotify_client, database)
+
+        print("[Discover Refresh] Starting forced refresh of discover data...")
+
+        # Cache recent albums from watchlist and similar artists
+        print("[Discover Refresh] Caching recent albums...")
+        scanner.cache_discovery_recent_albums()
+
+        # Curate playlists
+        print("[Discover Refresh] Curating discovery playlists...")
+        scanner.curate_discovery_playlists()
+
+        # Get counts for response
+        active_source = _get_active_discovery_source()
+        recent_albums = database.get_discovery_recent_albums(limit=100, source=active_source)
+        release_radar = database.get_curated_playlist(f'release_radar_{active_source}') or []
+        discovery_weekly = database.get_curated_playlist(f'discovery_weekly_{active_source}') or []
+
+        print(f"[Discover Refresh] Complete! Recent albums: {len(recent_albums)}, Release Radar: {len(release_radar)} tracks, Discovery Weekly: {len(discovery_weekly)} tracks")
+
+        return jsonify({
+            "success": True,
+            "message": "Discover data refreshed",
+            "source": active_source,
+            "recent_albums_count": len(recent_albums),
+            "release_radar_tracks": len(release_radar),
+            "discovery_weekly_tracks": len(discovery_weekly)
+        })
+
+    except Exception as e:
+        print(f"Error refreshing discover data: {e}")
+        import traceback
+        traceback.print_exc()
+        return jsonify({"success": False, "error": str(e)}), 500
+
+
+@app.route('/api/discover/diagnose', methods=['GET'])
+def diagnose_discover_data():
+    """
+    Diagnostic endpoint to check the state of discover data.
+    Returns counts of similar artists, discovery pool, recent albums, etc.
+    """
+    try:
+        database = get_database()
+        active_source = _get_active_discovery_source()
+
+        with database._get_connection() as conn:
+            cursor = conn.cursor()
+
+            # Similar artists stats
+            cursor.execute("SELECT COUNT(*) as total FROM similar_artists")
+            total_similar = cursor.fetchone()['total']
+
+            cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_itunes_id IS NOT NULL")
+            similar_with_itunes = cursor.fetchone()['count']
+
+            cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_spotify_id IS NOT NULL")
+            similar_with_spotify = cursor.fetchone()['count']
+
+            # Discovery pool stats
+            cursor.execute("SELECT source, COUNT(*) as count FROM discovery_pool GROUP BY source")
+            pool_by_source = {row['source']: row['count'] for row in cursor.fetchall()}
+
+            # Recent albums stats
+            cursor.execute("SELECT source, COUNT(*) as count FROM discovery_recent_albums GROUP BY source")
+            albums_by_source = {row['source']: row['count'] for row in cursor.fetchall()}
+
+            # Curated playlists
+            cursor.execute("SELECT playlist_type, track_ids_json FROM discovery_curated_playlists")
+            playlists = {}
+            for row in cursor.fetchall():
+                import json
+                track_ids = json.loads(row['track_ids_json']) if row['track_ids_json'] else []
+                playlists[row['playlist_type']] = len(track_ids)
+
+            # Watchlist artists
+            cursor.execute("SELECT COUNT(*) as total FROM watchlist_artists")
+            total_watchlist = cursor.fetchone()['total']
+
+            cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE itunes_artist_id IS NOT NULL")
+            watchlist_with_itunes = cursor.fetchone()['count']
+
+        return jsonify({
+            "success": True,
+            "active_source": active_source,
+            "similar_artists": {
+                "total": total_similar,
+                "with_itunes_id": similar_with_itunes,
+                "with_spotify_id": similar_with_spotify
+            },
+            "discovery_pool": pool_by_source,
+            "recent_albums": albums_by_source,
+            "curated_playlists": playlists,
+            "watchlist_artists": {
+                "total": total_watchlist,
+                "with_itunes_id": watchlist_with_itunes
+            }
+        })
+
+    except Exception as e:
+        print(f"Error diagnosing discover data: {e}")
+        return jsonify({"success": False, "error": str(e)}), 500
+
+
 # ========================================
 # SEASONAL DISCOVERY ENDPOINTS
 # ========================================

webui/static/script.js

@@ -33121,8 +33121,16 @@ async function openDownloadModalForRecentAlbum(albumIndex) {
     showLoadingOverlay(`Loading tracks for ${album.album_name}...`);
 
     try {
-        // Fetch album tracks from Spotify API via backend
+        // Determine source and album ID - use source-agnostic endpoint
-        const response = await fetch(`/api/spotify/album/${album.album_spotify_id}`);
+        const source = album.source || (album.album_spotify_id ? 'spotify' : 'itunes');
+        const albumId = source === 'spotify' ? album.album_spotify_id : album.album_itunes_id;
+
+        if (!albumId) {
+            throw new Error(`No ${source} album ID available`);
+        }
+
+        // Fetch album tracks from appropriate source via backend
+        const response = await fetch(`/api/discover/album/${source}/${albumId}`);
         if (!response.ok) {
             throw new Error('Failed to fetch album tracks');
         }
@@ -33157,13 +33165,14 @@ async function openDownloadModalForRecentAlbum(albumIndex) {
             };
         });
 
-        // Create virtual playlist ID
+        // Create virtual playlist ID using the appropriate album ID
-        const virtualPlaylistId = `discover_album_${album.album_spotify_id}`;
+        const virtualPlaylistId = `discover_album_${albumId}`;
 
         // CRITICAL FIX: Pass proper artist/album context for modal display
         const artistContext = {
-            id: album.artist_spotify_id,
+            id: source === 'spotify' ? album.artist_spotify_id : album.artist_itunes_id,
-            name: album.artist_name
+            name: album.artist_name,
+            source: source
         };
 
         const albumContext = {