Fix iTunes-only Discover page not loading data

- Add similar artists fetching to web UI scan loop
  - Add database migration for UNIQUE constraint on similar_artists table                                          - Add source-agnostic /api/discover/album endpoint for iTunes support
  - Fix NOT NULL constraint on discovery_recent_albums blocking iTunes albums
  - Add fallback to watchlist artists when no similar artists exist
  - Add /api/discover/refresh and /api/discover/diagnose endpoints
  - Add retry logic with exponential backoff for iTunes API calls
  - Ensure cache_discovery_recent_albums runs even when pool population skips
This commit is contained in:
Broque Thomas 2026-01-24 21:48:17 -08:00
parent b0e34c6942
commit 3cb88669e3
5 changed files with 808 additions and 34 deletions

View file

@ -21,9 +21,61 @@ logger = get_logger("watchlist_scanner")
# Rate limiting constants for watchlist operations
DELAY_BETWEEN_ARTISTS = 2.0 # 2 seconds between different artists
DELAY_BETWEEN_ALBUMS = 0.5 # 500ms between albums for same artist
DELAY_BETWEEN_ALBUMS = 0.5 # 500ms between albums for same artist
DELAY_BETWEEN_API_BATCHES = 1.0 # 1 second between API batch operations
# iTunes API retry configuration
ITUNES_MAX_RETRIES = 3
ITUNES_BASE_DELAY = 1.0 # Base delay in seconds for exponential backoff
def itunes_api_call_with_retry(func, *args, max_retries=ITUNES_MAX_RETRIES, **kwargs):
"""
Execute an iTunes API call with exponential backoff retry logic.
Args:
func: The function to call
*args: Arguments to pass to the function
max_retries: Maximum number of retry attempts
**kwargs: Keyword arguments to pass to the function
Returns:
The result of the function call, or None if all retries failed
"""
last_error = None
for attempt in range(max_retries):
try:
result = func(*args, **kwargs)
return result
except requests.exceptions.HTTPError as e:
# Handle rate limiting (429) and server errors (5xx)
if e.response is not None and e.response.status_code == 429:
delay = ITUNES_BASE_DELAY * (2 ** attempt)
logger.warning(f"[iTunes] Rate limited, retrying in {delay}s (attempt {attempt + 1}/{max_retries})")
time.sleep(delay)
last_error = e
elif e.response is not None and e.response.status_code >= 500:
delay = ITUNES_BASE_DELAY * (2 ** attempt)
logger.warning(f"[iTunes] Server error {e.response.status_code}, retrying in {delay}s (attempt {attempt + 1}/{max_retries})")
time.sleep(delay)
last_error = e
else:
raise # Don't retry on client errors (4xx except 429)
except requests.exceptions.RequestException as e:
# Retry on connection errors
delay = ITUNES_BASE_DELAY * (2 ** attempt)
logger.warning(f"[iTunes] Connection error, retrying in {delay}s (attempt {attempt + 1}/{max_retries}): {e}")
time.sleep(delay)
last_error = e
except Exception as e:
# Don't retry on other exceptions
raise
if last_error:
logger.error(f"[iTunes] All {max_retries} retry attempts failed: {last_error}")
return None
def clean_track_name_for_search(track_name):
"""
Intelligently cleans a track name for searching by removing noise while preserving important version information.
@ -1287,9 +1339,11 @@ class WatchlistScanner:
except Exception as e:
logger.debug(f"Spotify match failed for {artist_name_to_match}: {e}")
# Try to match on iTunes
# Try to match on iTunes (with retry for rate limiting)
try:
itunes_results = itunes_client.search_artists(artist_name_to_match, limit=1)
itunes_results = itunes_api_call_with_retry(
itunes_client.search_artists, artist_name_to_match, limit=1
)
if itunes_results and len(itunes_results) > 0:
itunes_artist = itunes_results[0]
# Skip if this is the searched artist
@ -1301,8 +1355,10 @@ class WatchlistScanner:
# Use iTunes genres if we don't have Spotify genres
if not artist_data['genres'] and hasattr(itunes_artist, 'genres'):
artist_data['genres'] = itunes_artist.genres
else:
logger.info(f" [iTunes] No match found for: {artist_name_to_match}")
except Exception as e:
logger.debug(f"iTunes match failed for {artist_name_to_match}: {e}")
logger.info(f" [iTunes] Match failed for {artist_name_to_match}: {e}")
# Only add if we got at least one ID
if artist_data['spotify_id'] or artist_data['itunes_id']:
@ -1314,7 +1370,11 @@ class WatchlistScanner:
logger.debug(f"Error matching {artist_name_to_match}: {match_error}")
continue
logger.info(f"Matched {len(matched_artists)} similar artists (Spotify + iTunes)")
# Log detailed matching statistics
itunes_matched = sum(1 for a in matched_artists if a.get('itunes_id'))
spotify_matched = sum(1 for a in matched_artists if a.get('spotify_id'))
both_matched = sum(1 for a in matched_artists if a.get('itunes_id') and a.get('spotify_id'))
logger.info(f"Matched {len(matched_artists)} similar artists - iTunes: {itunes_matched}, Spotify: {spotify_matched}, Both: {both_matched}")
return matched_artists
except requests.exceptions.RequestException as e:
@ -1436,9 +1496,15 @@ class WatchlistScanner:
from datetime import datetime, timedelta
import random
# Check if we should run (prevents over-polling)
if not self.database.should_populate_discovery_pool(hours_threshold=24):
logger.info("Discovery pool was populated recently (< 24 hours ago). Skipping.")
# Check if we should run discovery pool population (prevents over-polling)
skip_pool_population = not self.database.should_populate_discovery_pool(hours_threshold=24)
if skip_pool_population:
logger.info("Discovery pool was populated recently (< 24 hours ago). Skipping pool population.")
logger.info("But still refreshing recent albums cache and curated playlists...")
# Still run these even when skipping main pool population
self.cache_discovery_recent_albums()
self.curate_discovery_playlists()
return
logger.info("Populating discovery pool from similar artists...")
@ -1461,7 +1527,11 @@ class WatchlistScanner:
similar_artists = self.database.get_top_similar_artists(limit=top_artists_limit)
if not similar_artists:
logger.info("No similar artists found to populate discovery pool")
logger.info("No similar artists found to populate discovery pool from similar artists")
logger.info("But still caching recent albums from watchlist artists and curating playlists...")
# Still run these even without similar artists - they use watchlist artists
self.cache_discovery_recent_albums()
self.curate_discovery_playlists()
return
logger.info(f"Processing {len(similar_artists)} top similar artists for discovery pool")
@ -2056,26 +2126,40 @@ class WatchlistScanner:
logger.debug(f"Error processing album: {e}")
return False
# Track resolution stats
itunes_resolved = 0
itunes_failed_resolve = 0
# Process watchlist artists
for artist in watchlist_artists:
# Always process iTunes (baseline)
itunes_id = artist.itunes_artist_id
if not itunes_id:
# Try to resolve iTunes ID on-the-fly
# Try to resolve iTunes ID on-the-fly (with retry for rate limiting)
try:
results = itunes_client.search_artists(artist.artist_name, limit=1)
results = itunes_api_call_with_retry(
itunes_client.search_artists, artist.artist_name, limit=1
)
if results and len(results) > 0:
itunes_id = results[0].id
except:
pass
itunes_resolved += 1
logger.debug(f"[iTunes] Resolved ID for {artist.artist_name}: {itunes_id}")
else:
itunes_failed_resolve += 1
logger.info(f"[iTunes] No artist found for: {artist.artist_name}")
except Exception as e:
itunes_failed_resolve += 1
logger.info(f"[iTunes] Failed to resolve {artist.artist_name}: {e}")
if itunes_id:
try:
albums = itunes_client.get_artist_albums(itunes_id, album_type='album,single', limit=20)
albums = itunes_api_call_with_retry(
itunes_client.get_artist_albums, itunes_id, album_type='album,single', limit=20
)
for album in albums or []:
process_album(album, artist.artist_name, artist.spotify_artist_id, itunes_id, 'itunes')
except Exception as e:
logger.debug(f"Error fetching iTunes albums for {artist.artist_name}: {e}")
logger.info(f"[iTunes] Error fetching albums for {artist.artist_name}: {e}")
# Process Spotify if authenticated
if spotify_available and artist.spotify_artist_id:
@ -2097,23 +2181,33 @@ class WatchlistScanner:
# Always process iTunes (baseline)
itunes_id = artist.similar_artist_itunes_id
if not itunes_id:
# Try to resolve iTunes ID on-the-fly
# Try to resolve iTunes ID on-the-fly (with retry for rate limiting)
try:
results = itunes_client.search_artists(artist.similar_artist_name, limit=1)
results = itunes_api_call_with_retry(
itunes_client.search_artists, artist.similar_artist_name, limit=1
)
if results and len(results) > 0:
itunes_id = results[0].id
# Cache for future
self.database.update_similar_artist_itunes_id(artist.id, itunes_id)
except:
pass
itunes_resolved += 1
logger.debug(f"[iTunes] Resolved ID for similar artist {artist.similar_artist_name}: {itunes_id}")
else:
itunes_failed_resolve += 1
logger.info(f"[iTunes] No artist found for similar: {artist.similar_artist_name}")
except Exception as e:
itunes_failed_resolve += 1
logger.info(f"[iTunes] Failed to resolve similar {artist.similar_artist_name}: {e}")
if itunes_id:
try:
albums = itunes_client.get_artist_albums(itunes_id, album_type='album,single', limit=20)
albums = itunes_api_call_with_retry(
itunes_client.get_artist_albums, itunes_id, album_type='album,single', limit=20
)
for album in albums or []:
process_album(album, artist.similar_artist_name, artist.similar_artist_spotify_id, itunes_id, 'itunes')
except Exception as e:
logger.debug(f"Error fetching iTunes albums for {artist.similar_artist_name}: {e}")
logger.info(f"[iTunes] Error fetching albums for similar {artist.similar_artist_name}: {e}")
# Process Spotify if authenticated
if spotify_available and artist.similar_artist_spotify_id:
@ -2132,6 +2226,7 @@ class WatchlistScanner:
total_cached = cached_count['spotify'] + cached_count['itunes']
logger.info(f"Cached {total_cached} recent albums (Spotify: {cached_count['spotify']}, iTunes: {cached_count['itunes']}) from {albums_checked} albums checked")
logger.info(f"[iTunes] ID resolution stats: {itunes_resolved} resolved, {itunes_failed_resolve} failed")
except Exception as e:
logger.error(f"Error caching discovery recent albums: {e}")
@ -2171,6 +2266,9 @@ class WatchlistScanner:
recent_albums = self.database.get_discovery_recent_albums(limit=50, source=source)
release_radar_tracks = []
if not recent_albums:
logger.warning(f"[{source.upper()}] No recent albums found for Release Radar - check cache_discovery_recent_albums()")
if recent_albums:
# Group albums by artist for variety
albums_by_artist = {}
@ -2196,7 +2294,9 @@ class WatchlistScanner:
if source == 'spotify':
album_data = self.spotify_client.get_album(album_id)
else:
album_data = itunes_client.get_album(album_id)
album_data = itunes_api_call_with_retry(
itunes_client.get_album, album_id
)
if not album_data or 'tracks' not in album_data:
continue
@ -2295,6 +2395,9 @@ class WatchlistScanner:
logger.info(f"Curating Discovery Weekly for {source}...")
discovery_tracks = self.database.get_discovery_pool_tracks(limit=2000, new_releases_only=False, source=source)
if not discovery_tracks:
logger.warning(f"[{source.upper()}] No discovery pool tracks found for Discovery Weekly - check populate_discovery_pool()")
discovery_weekly_tracks = []
if discovery_tracks:
# Separate tracks by popularity tiers

View file

@ -636,6 +636,90 @@ class MusicDatabase:
cursor.execute("ALTER TABLE discovery_recent_albums ADD COLUMN source TEXT DEFAULT 'spotify'")
logger.info("Added iTunes columns to discovery_recent_albums table for dual-source discovery")
# Migration: Fix NOT NULL constraint on album_spotify_id (required for iTunes-only albums)
# Check if album_spotify_id has NOT NULL constraint by checking table schema
cursor.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='discovery_recent_albums'")
table_schema = cursor.fetchone()
if table_schema and 'album_spotify_id TEXT NOT NULL' in (table_schema[0] or ''):
logger.info("Migrating discovery_recent_albums to allow NULL album_spotify_id for iTunes support...")
# SQLite doesn't support ALTER COLUMN, so recreate table
cursor.execute("""
CREATE TABLE IF NOT EXISTS discovery_recent_albums_new (
id INTEGER PRIMARY KEY AUTOINCREMENT,
album_spotify_id TEXT,
album_itunes_id TEXT,
artist_spotify_id TEXT,
artist_itunes_id TEXT,
source TEXT NOT NULL DEFAULT 'spotify',
album_name TEXT NOT NULL,
artist_name TEXT NOT NULL,
album_cover_url TEXT,
release_date TEXT NOT NULL,
album_type TEXT DEFAULT 'album',
cached_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(album_spotify_id, album_itunes_id, source)
)
""")
cursor.execute("""
INSERT OR IGNORE INTO discovery_recent_albums_new
SELECT * FROM discovery_recent_albums
""")
cursor.execute("DROP TABLE discovery_recent_albums")
cursor.execute("ALTER TABLE discovery_recent_albums_new RENAME TO discovery_recent_albums")
conn.commit()
logger.info("Successfully migrated discovery_recent_albums table for iTunes support")
# Migration: Add UNIQUE constraint to similar_artists table
# Test if ON CONFLICT works by trying a dummy operation
needs_similar_migration = False
try:
cursor.execute("""
INSERT INTO similar_artists
(source_artist_id, similar_artist_name, similarity_rank, occurrence_count, last_updated)
VALUES ('__migration_test__', '__migration_test__', 1, 1, CURRENT_TIMESTAMP)
ON CONFLICT(source_artist_id, similar_artist_name)
DO UPDATE SET occurrence_count = occurrence_count
""")
# Clean up test row
cursor.execute("DELETE FROM similar_artists WHERE source_artist_id = '__migration_test__'")
logger.info("similar_artists table has correct UNIQUE constraint")
except Exception as constraint_error:
logger.info(f"similar_artists needs migration (constraint test failed: {constraint_error})")
needs_similar_migration = True
if needs_similar_migration:
logger.info("Migrating similar_artists to add UNIQUE constraint...")
# Get a fresh connection for the migration
with self._get_connection() as migration_conn:
migration_cursor = migration_conn.cursor()
# SQLite doesn't support adding constraints, so recreate table
migration_cursor.execute("DROP TABLE IF EXISTS similar_artists_new")
migration_cursor.execute("""
CREATE TABLE similar_artists_new (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_artist_id TEXT NOT NULL,
similar_artist_spotify_id TEXT,
similar_artist_itunes_id TEXT,
similar_artist_name TEXT NOT NULL,
similarity_rank INTEGER DEFAULT 1,
occurrence_count INTEGER DEFAULT 1,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(source_artist_id, similar_artist_name)
)
""")
migration_cursor.execute("""
INSERT OR IGNORE INTO similar_artists_new
(source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id,
similar_artist_name, similarity_rank, occurrence_count, last_updated)
SELECT source_artist_id, similar_artist_spotify_id, similar_artist_itunes_id,
similar_artist_name, similarity_rank, occurrence_count, last_updated
FROM similar_artists
""")
migration_cursor.execute("DROP TABLE similar_artists")
migration_cursor.execute("ALTER TABLE similar_artists_new RENAME TO similar_artists")
migration_conn.commit()
logger.info("Successfully migrated similar_artists table with UNIQUE constraint")
# ============== INDEXES (after migrations to ensure columns exist) ==============
cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_source ON similar_artists (source_artist_id)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_spotify ON similar_artists (similar_artist_spotify_id)")

View file

@ -0,0 +1,270 @@
#!/usr/bin/env python3
"""
Diagnostic script to check iTunes data availability for the Discover page.
Run this script to identify issues with iTunes data population:
- Similar artists missing iTunes IDs
- Discovery pool tracks by source
- Recent albums by source
- Curated playlists status
Usage:
python tools/diagnose_itunes_discover.py
"""
import sys
import os
import json
# Add parent directory to path for imports
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from database.music_database import MusicDatabase
def diagnose_itunes_discover():
"""Run diagnostic checks for iTunes discover data."""
print("=" * 60)
print("iTunes Discover Page Diagnostic Report")
print("=" * 60)
db = MusicDatabase()
# 1. Check Similar Artists
print("\n[1] SIMILAR ARTISTS")
print("-" * 40)
try:
with db._get_connection() as conn:
cursor = conn.cursor()
# Total similar artists
cursor.execute("SELECT COUNT(*) as total FROM similar_artists")
total = cursor.fetchone()['total']
# With iTunes IDs
cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_itunes_id IS NOT NULL")
with_itunes = cursor.fetchone()['count']
# With Spotify IDs
cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_spotify_id IS NOT NULL")
with_spotify = cursor.fetchone()['count']
# With both
cursor.execute("""
SELECT COUNT(*) as count FROM similar_artists
WHERE similar_artist_itunes_id IS NOT NULL
AND similar_artist_spotify_id IS NOT NULL
""")
with_both = cursor.fetchone()['count']
print(f" Total similar artists: {total}")
print(f" With iTunes ID: {with_itunes} ({100*with_itunes/total:.1f}%)" if total > 0 else " With iTunes ID: 0")
print(f" With Spotify ID: {with_spotify} ({100*with_spotify/total:.1f}%)" if total > 0 else " With Spotify ID: 0")
print(f" With BOTH IDs: {with_both} ({100*with_both/total:.1f}%)" if total > 0 else " With BOTH IDs: 0")
if with_itunes == 0 and total > 0:
print(" [CRITICAL] No similar artists have iTunes IDs - Hero section will be empty!")
elif with_itunes < total * 0.5:
print(" [WARNING] Less than 50% of similar artists have iTunes IDs")
else:
print(" [OK] iTunes coverage is adequate")
except Exception as e:
print(f" [ERROR] Could not check similar artists: {e}")
# 2. Check Discovery Pool
print("\n[2] DISCOVERY POOL")
print("-" * 40)
try:
with db._get_connection() as conn:
cursor = conn.cursor()
# Total tracks
cursor.execute("SELECT COUNT(*) as total FROM discovery_pool")
total = cursor.fetchone()['total']
# By source
cursor.execute("""
SELECT source, COUNT(*) as count
FROM discovery_pool
GROUP BY source
""")
source_counts = {row['source']: row['count'] for row in cursor.fetchall()}
print(f" Total tracks: {total}")
print(f" Spotify tracks: {source_counts.get('spotify', 0)}")
print(f" iTunes tracks: {source_counts.get('itunes', 0)}")
if source_counts.get('itunes', 0) == 0 and total > 0:
print(" [CRITICAL] No iTunes tracks in discovery pool - Fresh Tape/Archives will be empty!")
elif source_counts.get('itunes', 0) < total * 0.3:
print(" [WARNING] Low iTunes track count in discovery pool")
else:
print(" [OK] iTunes tracks present")
except Exception as e:
print(f" [ERROR] Could not check discovery pool: {e}")
# 3. Check Recent Albums
print("\n[3] RECENT ALBUMS CACHE")
print("-" * 40)
try:
with db._get_connection() as conn:
cursor = conn.cursor()
# Total albums
cursor.execute("SELECT COUNT(*) as total FROM discovery_recent_albums")
total = cursor.fetchone()['total']
# By source
cursor.execute("""
SELECT source, COUNT(*) as count
FROM discovery_recent_albums
GROUP BY source
""")
source_counts = {row['source']: row['count'] for row in cursor.fetchall()}
print(f" Total recent albums: {total}")
print(f" Spotify albums: {source_counts.get('spotify', 0)}")
print(f" iTunes albums: {source_counts.get('itunes', 0)}")
if source_counts.get('itunes', 0) == 0 and total > 0:
print(" [CRITICAL] No iTunes albums cached - Recent Releases section will be empty!")
elif source_counts.get('itunes', 0) < 5:
print(" [WARNING] Very few iTunes albums cached")
else:
print(" [OK] iTunes albums cached")
except Exception as e:
print(f" [ERROR] Could not check recent albums: {e}")
# 4. Check Curated Playlists
print("\n[4] CURATED PLAYLISTS")
print("-" * 40)
try:
with db._get_connection() as conn:
cursor = conn.cursor()
playlists_to_check = [
'release_radar',
'release_radar_spotify',
'release_radar_itunes',
'discovery_weekly',
'discovery_weekly_spotify',
'discovery_weekly_itunes'
]
for playlist_type in playlists_to_check:
cursor.execute("""
SELECT track_ids_json FROM discovery_curated_playlists
WHERE playlist_type = ?
""", (playlist_type,))
row = cursor.fetchone()
if row:
track_ids = json.loads(row['track_ids_json'])
status = f"{len(track_ids)} tracks"
if len(track_ids) == 0:
status += " [EMPTY]"
else:
status = "[NOT FOUND]"
print(f" {playlist_type}: {status}")
# Check iTunes-specific playlists
cursor.execute("""
SELECT track_ids_json FROM discovery_curated_playlists
WHERE playlist_type = 'release_radar_itunes'
""")
itunes_rr = cursor.fetchone()
cursor.execute("""
SELECT track_ids_json FROM discovery_curated_playlists
WHERE playlist_type = 'discovery_weekly_itunes'
""")
itunes_dw = cursor.fetchone()
if not itunes_rr or len(json.loads(itunes_rr['track_ids_json'])) == 0:
print("\n [CRITICAL] release_radar_itunes is empty or missing!")
if not itunes_dw or len(json.loads(itunes_dw['track_ids_json'])) == 0:
print(" [CRITICAL] discovery_weekly_itunes is empty or missing!")
except Exception as e:
print(f" [ERROR] Could not check curated playlists: {e}")
# 5. Check Watchlist Artists
print("\n[5] WATCHLIST ARTISTS")
print("-" * 40)
try:
with db._get_connection() as conn:
cursor = conn.cursor()
# Total artists
cursor.execute("SELECT COUNT(*) as total FROM watchlist_artists")
total = cursor.fetchone()['total']
# With iTunes IDs
cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE itunes_artist_id IS NOT NULL")
with_itunes = cursor.fetchone()['count']
# With Spotify IDs
cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE spotify_artist_id IS NOT NULL")
with_spotify = cursor.fetchone()['count']
print(f" Total watchlist artists: {total}")
print(f" With iTunes ID: {with_itunes} ({100*with_itunes/total:.1f}%)" if total > 0 else " With iTunes ID: 0")
print(f" With Spotify ID: {with_spotify} ({100*with_spotify/total:.1f}%)" if total > 0 else " With Spotify ID: 0")
if with_itunes == 0 and total > 0:
print(" [WARNING] No watchlist artists have iTunes IDs - source artist data limited")
except Exception as e:
print(f" [ERROR] Could not check watchlist artists: {e}")
# Summary
print("\n" + "=" * 60)
print("SUMMARY & RECOMMENDED ACTIONS")
print("=" * 60)
print("""
If you see [CRITICAL] or [WARNING] messages above, follow these steps:
QUICK FIX - Force Refresh Discover Data:
-----------------------------------------
Call the API endpoint to refresh discover data:
curl -X POST http://localhost:5000/api/discover/refresh
This will:
- Cache recent albums from your watchlist artists
- Create curated playlists (Release Radar & Discovery Weekly)
FULL FIX - Run Watchlist Scan:
------------------------------
1. Go to the web UI Settings page
2. Click "Scan Watchlist" button
3. Wait for scan to complete
This will:
- Fetch similar artists from MusicMap for each watchlist artist
- Populate the discovery pool with tracks
- Cache recent albums
- Create curated playlists
ROOT CAUSE NOTES:
-----------------
- Similar artists = 0: MusicMap fetch may have failed. Watchlist scan needed.
- Recent albums = 0: cache_discovery_recent_albums() needs to run.
- Curated playlists missing: curate_discovery_playlists() needs to run.
The discover page will now fall back to watchlist artists if similar
artists are not available, so basic functionality should still work.
""")
if __name__ == '__main__':
diagnose_itunes_discover()

View file

@ -14327,6 +14327,105 @@ def search_itunes_tracks():
return jsonify({"error": str(e)}), 500
@app.route('/api/itunes/album/<album_id>', methods=['GET'])
def get_itunes_album_tracks(album_id):
"""Fetches full track details for a specific iTunes album."""
try:
from core.itunes_client import iTunesClient
itunes_client = iTunesClient()
album_data = itunes_client.get_album(album_id)
if not album_data:
return jsonify({"error": "Album not found"}), 404
# Get tracks for this album
tracks_data = itunes_client.get_album_tracks(album_id)
tracks = tracks_data.get('items', []) if tracks_data else []
# Format response to match Spotify structure for frontend compatibility
album_dict = {
'id': album_data.get('id', album_id),
'name': album_data.get('name', 'Unknown Album'),
'artists': album_data.get('artists', []),
'release_date': album_data.get('release_date', ''),
'total_tracks': album_data.get('total_tracks', len(tracks)),
'album_type': album_data.get('album_type', 'album'),
'images': album_data.get('images', []),
'tracks': tracks,
'source': 'itunes'
}
return jsonify(album_dict)
except Exception as e:
logger.error(f"Error fetching iTunes album tracks: {e}")
return jsonify({"error": str(e)}), 500
@app.route('/api/discover/album/<source>/<album_id>', methods=['GET'])
def get_discover_album(source, album_id):
"""
Source-agnostic album endpoint for discover page.
Fetches album from the appropriate source (spotify or itunes).
"""
try:
if source == 'spotify':
if not spotify_client or not spotify_client.is_authenticated():
return jsonify({"error": "Spotify not authenticated."}), 401
album_data = spotify_client.get_album(album_id)
if not album_data:
return jsonify({"error": "Album not found"}), 404
tracks = album_data.get('tracks', {}).get('items', [])
if not tracks:
tracks_data = spotify_client.get_album_tracks(album_id)
if tracks_data and 'items' in tracks_data:
tracks = tracks_data['items']
return jsonify({
'id': album_data['id'],
'name': album_data['name'],
'artists': album_data.get('artists', []),
'release_date': album_data.get('release_date', ''),
'total_tracks': album_data.get('total_tracks', 0),
'album_type': album_data.get('album_type', 'album'),
'images': album_data.get('images', []),
'tracks': tracks,
'source': 'spotify'
})
elif source == 'itunes':
from core.itunes_client import iTunesClient
itunes_client = iTunesClient()
album_data = itunes_client.get_album(album_id)
if not album_data:
return jsonify({"error": "Album not found"}), 404
tracks_data = itunes_client.get_album_tracks(album_id)
tracks = tracks_data.get('items', []) if tracks_data else []
return jsonify({
'id': album_data.get('id', album_id),
'name': album_data.get('name', 'Unknown Album'),
'artists': album_data.get('artists', []),
'release_date': album_data.get('release_date', ''),
'total_tracks': album_data.get('total_tracks', len(tracks)),
'album_type': album_data.get('album_type', 'album'),
'images': album_data.get('images', []),
'tracks': tracks,
'source': 'itunes'
})
else:
return jsonify({"error": f"Unknown source: {source}"}), 400
except Exception as e:
logger.error(f"Error fetching discover album: {e}")
return jsonify({"error": str(e)}), 500
# ===================================================================
# TIDAL PLAYLIST API ENDPOINTS
# ===================================================================
@ -17618,7 +17717,24 @@ def start_watchlist_scan():
})())
print(f"✅ Scanned {artist.artist_name}: {artist_new_tracks} new tracks found, {artist_added_tracks} added to wishlist")
# Fetch similar artists for discovery feature
# This is critical for the discover page to work
try:
watchlist_scan_state['current_phase'] = 'fetching_similar_artists'
source_artist_id = artist.spotify_artist_id or artist.itunes_artist_id or str(artist.id)
if database.has_fresh_similar_artists(source_artist_id, days_threshold=30):
print(f" Similar artists for {artist.artist_name} are cached and fresh")
# Still backfill missing iTunes IDs
scanner._backfill_similar_artists_itunes_ids(source_artist_id)
else:
print(f" Fetching similar artists for {artist.artist_name}...")
scanner.update_similar_artists(artist)
print(f" Similar artists updated for {artist.artist_name}")
except Exception as similar_error:
print(f" ⚠️ Failed to update similar artists for {artist.artist_name}: {similar_error}")
# Delay between artists
if i < len(watchlist_artists) - 1:
watchlist_scan_state['current_phase'] = 'rate_limiting'
@ -18469,11 +18585,63 @@ def get_discover_hero():
active_source = _get_active_discovery_source()
print(f"🎵 Discover hero using source: {active_source}")
# Import iTunes client for fallback
from core.itunes_client import iTunesClient
itunes_client = iTunesClient()
# Get top similar artists (by occurrence count) - get 20 for variety
similar_artists = database.get_top_similar_artists(limit=20)
# FALLBACK: If no similar artists exist, use watchlist artists for Hero section
if not similar_artists:
return jsonify({"success": True, "artists": [], "source": active_source})
print("[Discover Hero] No similar artists found, falling back to watchlist artists")
watchlist_artists = database.get_watchlist_artists()
if not watchlist_artists:
return jsonify({"success": True, "artists": [], "source": active_source})
# Convert watchlist artists to hero format
import random
shuffled_watchlist = list(watchlist_artists)
random.shuffle(shuffled_watchlist)
hero_artists = []
for artist in shuffled_watchlist[:10]:
artist_id = artist.itunes_artist_id if active_source == 'itunes' else artist.spotify_artist_id
if not artist_id:
continue
artist_data = {
"spotify_artist_id": artist.spotify_artist_id,
"itunes_artist_id": artist.itunes_artist_id,
"artist_id": artist_id,
"artist_name": artist.artist_name,
"occurrence_count": 1,
"similarity_rank": 1,
"source": active_source,
"is_watchlist": True
}
# Try to get artist image
try:
if active_source == 'itunes' and artist.itunes_artist_id:
itunes_artist = itunes_client.get_artist(artist.itunes_artist_id)
if itunes_artist:
artist_data['image_url'] = itunes_artist.get('images', [{}])[0].get('url') if itunes_artist.get('images') else None
artist_data['genres'] = itunes_artist.get('genres', [])
elif active_source == 'spotify' and artist.spotify_artist_id:
if spotify_client and spotify_client.is_authenticated():
sp_artist = spotify_client.get_artist(artist.spotify_artist_id)
if sp_artist and sp_artist.get('images'):
artist_data['image_url'] = sp_artist['images'][0]['url'] if sp_artist['images'] else None
artist_data['genres'] = sp_artist.get('genres', [])
except Exception as img_err:
print(f"Could not fetch watchlist artist image: {img_err}")
hero_artists.append(artist_data)
print(f"[Discover Hero] Returning {len(hero_artists)} watchlist artists as fallback")
return jsonify({"success": True, "artists": hero_artists, "source": active_source, "fallback": "watchlist"})
# Filter to artists that have the appropriate ID for the active source
valid_artists = []
@ -18486,16 +18654,41 @@ def get_discover_hero():
elif artist.similar_artist_spotify_id and artist.similar_artist_itunes_id:
valid_artists.append(artist)
# FALLBACK: If no valid artists for iTunes, try to resolve iTunes IDs on-the-fly
if active_source == 'itunes' and not valid_artists:
print(f"[iTunes Fallback] No artists with iTunes IDs found, attempting on-the-fly resolution for {len(similar_artists)} artists")
resolved_count = 0
for artist in similar_artists:
if artist.similar_artist_itunes_id:
valid_artists.append(artist)
continue
# Try to resolve iTunes ID by name
try:
itunes_results = itunes_client.search_artists(artist.similar_artist_name, limit=1)
if itunes_results and len(itunes_results) > 0:
itunes_id = itunes_results[0].id
# Cache the resolved ID for future use
database.update_similar_artist_itunes_id(artist.id, itunes_id)
# Create a modified artist object with the resolved ID
artist.similar_artist_itunes_id = itunes_id
valid_artists.append(artist)
resolved_count += 1
print(f" [Resolved] {artist.similar_artist_name} -> iTunes ID: {itunes_id}")
except Exception as resolve_err:
print(f" [Failed] Could not resolve iTunes ID for {artist.similar_artist_name}: {resolve_err}")
# Stop after 10 successful resolutions to avoid rate limiting
if len(valid_artists) >= 10:
break
print(f"[iTunes Fallback] Resolved {resolved_count} artists with iTunes IDs")
print(f"[Discover Hero] Found {len(valid_artists)} valid artists for source: {active_source}")
# Shuffle for variety and take top 10
import random
shuffled = list(valid_artists)
random.shuffle(shuffled)
similar_artists = shuffled[:10]
# Import iTunes client for fallback
from core.itunes_client import iTunesClient
itunes_client = iTunesClient()
# Convert to JSON format with data enrichment from appropriate source
hero_artists = []
for artist in similar_artists:
@ -18685,6 +18878,121 @@ def get_discover_weekly():
print(f"Error getting discovery weekly: {e}")
return jsonify({"success": False, "error": str(e)}), 500
@app.route('/api/discover/refresh', methods=['POST'])
def refresh_discover_data():
"""
Force refresh discover page data (recent albums cache and curated playlists).
Useful for initial setup or when data appears stale.
"""
try:
from core.watchlist_scanner import WatchlistScanner
database = get_database()
scanner = WatchlistScanner(spotify_client, database)
print("[Discover Refresh] Starting forced refresh of discover data...")
# Cache recent albums from watchlist and similar artists
print("[Discover Refresh] Caching recent albums...")
scanner.cache_discovery_recent_albums()
# Curate playlists
print("[Discover Refresh] Curating discovery playlists...")
scanner.curate_discovery_playlists()
# Get counts for response
active_source = _get_active_discovery_source()
recent_albums = database.get_discovery_recent_albums(limit=100, source=active_source)
release_radar = database.get_curated_playlist(f'release_radar_{active_source}') or []
discovery_weekly = database.get_curated_playlist(f'discovery_weekly_{active_source}') or []
print(f"[Discover Refresh] Complete! Recent albums: {len(recent_albums)}, Release Radar: {len(release_radar)} tracks, Discovery Weekly: {len(discovery_weekly)} tracks")
return jsonify({
"success": True,
"message": "Discover data refreshed",
"source": active_source,
"recent_albums_count": len(recent_albums),
"release_radar_tracks": len(release_radar),
"discovery_weekly_tracks": len(discovery_weekly)
})
except Exception as e:
print(f"Error refreshing discover data: {e}")
import traceback
traceback.print_exc()
return jsonify({"success": False, "error": str(e)}), 500
@app.route('/api/discover/diagnose', methods=['GET'])
def diagnose_discover_data():
"""
Diagnostic endpoint to check the state of discover data.
Returns counts of similar artists, discovery pool, recent albums, etc.
"""
try:
database = get_database()
active_source = _get_active_discovery_source()
with database._get_connection() as conn:
cursor = conn.cursor()
# Similar artists stats
cursor.execute("SELECT COUNT(*) as total FROM similar_artists")
total_similar = cursor.fetchone()['total']
cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_itunes_id IS NOT NULL")
similar_with_itunes = cursor.fetchone()['count']
cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_spotify_id IS NOT NULL")
similar_with_spotify = cursor.fetchone()['count']
# Discovery pool stats
cursor.execute("SELECT source, COUNT(*) as count FROM discovery_pool GROUP BY source")
pool_by_source = {row['source']: row['count'] for row in cursor.fetchall()}
# Recent albums stats
cursor.execute("SELECT source, COUNT(*) as count FROM discovery_recent_albums GROUP BY source")
albums_by_source = {row['source']: row['count'] for row in cursor.fetchall()}
# Curated playlists
cursor.execute("SELECT playlist_type, track_ids_json FROM discovery_curated_playlists")
playlists = {}
for row in cursor.fetchall():
import json
track_ids = json.loads(row['track_ids_json']) if row['track_ids_json'] else []
playlists[row['playlist_type']] = len(track_ids)
# Watchlist artists
cursor.execute("SELECT COUNT(*) as total FROM watchlist_artists")
total_watchlist = cursor.fetchone()['total']
cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE itunes_artist_id IS NOT NULL")
watchlist_with_itunes = cursor.fetchone()['count']
return jsonify({
"success": True,
"active_source": active_source,
"similar_artists": {
"total": total_similar,
"with_itunes_id": similar_with_itunes,
"with_spotify_id": similar_with_spotify
},
"discovery_pool": pool_by_source,
"recent_albums": albums_by_source,
"curated_playlists": playlists,
"watchlist_artists": {
"total": total_watchlist,
"with_itunes_id": watchlist_with_itunes
}
})
except Exception as e:
print(f"Error diagnosing discover data: {e}")
return jsonify({"success": False, "error": str(e)}), 500
# ========================================
# SEASONAL DISCOVERY ENDPOINTS
# ========================================

View file

@ -33121,8 +33121,16 @@ async function openDownloadModalForRecentAlbum(albumIndex) {
showLoadingOverlay(`Loading tracks for ${album.album_name}...`);
try {
// Fetch album tracks from Spotify API via backend
const response = await fetch(`/api/spotify/album/${album.album_spotify_id}`);
// Determine source and album ID - use source-agnostic endpoint
const source = album.source || (album.album_spotify_id ? 'spotify' : 'itunes');
const albumId = source === 'spotify' ? album.album_spotify_id : album.album_itunes_id;
if (!albumId) {
throw new Error(`No ${source} album ID available`);
}
// Fetch album tracks from appropriate source via backend
const response = await fetch(`/api/discover/album/${source}/${albumId}`);
if (!response.ok) {
throw new Error('Failed to fetch album tracks');
}
@ -33157,13 +33165,14 @@ async function openDownloadModalForRecentAlbum(albumIndex) {
};
});
// Create virtual playlist ID
const virtualPlaylistId = `discover_album_${album.album_spotify_id}`;
// Create virtual playlist ID using the appropriate album ID
const virtualPlaylistId = `discover_album_${albumId}`;
// CRITICAL FIX: Pass proper artist/album context for modal display
const artistContext = {
id: album.artist_spotify_id,
name: album.artist_name
id: source === 'spotify' ? album.artist_spotify_id : album.artist_itunes_id,
name: album.artist_name,
source: source
};
const albumContext = {