Add universal metadata cache for Spotify & iTunes API responses with browsable dashboard tool

This commit is contained in:
Broque Thomas 2026-03-14 01:26:45 -07:00
parent c54e52e18d
commit 6de3ab7cef
8 changed files with 2454 additions and 44 deletions

View file

@ -5,6 +5,7 @@ import threading
from functools import wraps
from dataclasses import dataclass
from utils.logging_config import get_logger
from core.metadata_cache import get_metadata_cache
logger = get_logger("itunes_client")
@ -351,9 +352,22 @@ class iTunesClient:
@rate_limited
def search_tracks(self, query: str, limit: int = 20) -> List[Track]:
"""Search for tracks using iTunes API"""
# Check search cache
cache = get_metadata_cache()
cached_results = cache.get_search_results('itunes', 'track', query, limit)
if cached_results is not None:
tracks = []
for raw in cached_results:
try:
tracks.append(Track.from_itunes_track(raw))
except Exception:
pass
if tracks:
return tracks
results = self._search(query, 'song', limit)
tracks = []
# Collect artist IDs for batch lookup
artist_ids = set()
for track_data in results:
@ -361,19 +375,28 @@ class iTunesClient:
artist_id = str(track_data.get('artistId', ''))
if artist_id:
artist_ids.add(artist_id)
# Batch lookup artist clean names
clean_artist_map = {}
if artist_ids:
clean_artist_map = self._get_clean_artist_names(list(artist_ids))
raw_items = []
for track_data in results:
if track_data.get('wrapperType') == 'track' and track_data.get('kind') == 'song':
artist_id = str(track_data.get('artistId', ''))
clean_artist = clean_artist_map.get(artist_id)
track = Track.from_itunes_track(track_data, clean_artist_name=clean_artist)
tracks.append(track)
raw_items.append(track_data)
# Cache individual tracks + search mapping
entries = [(str(td.get('trackId', '')), td) for td in raw_items if td.get('trackId')]
if entries:
cache.store_entities_bulk('itunes', 'track', entries)
cache.store_search_results('itunes', 'track', query, limit,
[str(td.get('trackId', '')) for td in raw_items if td.get('trackId')])
return tracks
def _get_clean_artist_names(self, artist_ids: List[str]) -> Dict[str, str]:
@ -409,8 +432,43 @@ class iTunesClient:
def get_track_details(self, track_id: str) -> Optional[Dict[str, Any]]:
"""Get detailed track information including album data and track number"""
# Check cache for raw track data
cache = get_metadata_cache()
cached = cache.get_entity('itunes', 'track', str(track_id))
if cached:
# Reconstruct enhanced_data from cached raw track_data
# Try to get clean artist name from artist cache (avoids API call)
clean_artist_name = cached.get('artistName', 'Unknown Artist')
artist_id = str(cached.get('artistId', ''))
if artist_id:
cached_artist = cache.get_entity('itunes', 'artist', artist_id)
if cached_artist:
clean_artist_name = cached_artist.get('artistName', clean_artist_name)
# If no cached artist, use the track's artistName as-is (no API call)
return {
'id': str(cached.get('trackId', '')),
'name': cached.get('trackName', ''),
'track_number': cached.get('trackNumber', 0),
'disc_number': cached.get('discNumber', 1),
'duration_ms': cached.get('trackTimeMillis', 0),
'explicit': cached.get('trackExplicitness') == 'explicit',
'artists': [clean_artist_name],
'primary_artist': clean_artist_name,
'album': {
'id': str(cached.get('collectionId', '')),
'name': _clean_itunes_album_name(cached.get('collectionName', '')),
'total_tracks': cached.get('trackCount', 0),
'release_date': cached.get('releaseDate', ''),
'album_type': 'album',
'artists': [clean_artist_name]
},
'is_album_track': cached.get('trackCount', 0) > 1,
'raw_data': cached
}
results = self._lookup(id=track_id)
for track_data in results:
if track_data.get('wrapperType') == 'track':
# Enhance with additional useful metadata
@ -444,6 +502,8 @@ class iTunesClient:
'is_album_track': track_data.get('trackCount', 0) > 1,
'raw_data': track_data
}
# Cache the raw track data
cache.store_entity('itunes', 'track', str(track_id), track_data)
return enhanced_data
return None
@ -464,6 +524,19 @@ class iTunesClient:
Filters out clean versions when explicit versions are available.
"""
# Check search cache
cache = get_metadata_cache()
cached_results = cache.get_search_results('itunes', 'album', query, limit)
if cached_results is not None:
albums = []
for raw in cached_results:
try:
albums.append(Album.from_itunes_album(raw))
except Exception:
pass
if albums:
return albums
results = self._search(query, 'album', limit * 2) # Fetch more to account for filtering
albums = []
seen_albums = {} # Track albums by normalized name to prefer explicit versions
@ -489,12 +562,25 @@ class iTunesClient:
else:
seen_albums[key] = {'data': album_data, 'is_explicit': is_explicit}
# Convert to Album objects
# Convert to Album objects + collect raw items for caching
raw_items = []
for item in seen_albums.values():
album = Album.from_itunes_album(item['data'])
albums.append(album)
raw_items.append(item['data'])
return albums[:limit]
result = albums[:limit]
# Cache individual albums + search mapping
entries = [(str(ad.get('collectionId', '')), ad) for ad in raw_items if ad.get('collectionId')]
if entries:
cache.store_entities_bulk('itunes', 'album', entries)
# Only cache IDs for the albums we're actually returning
result_ids = [str(ad.get('collectionId', '')) for ad in raw_items[:limit] if ad.get('collectionId')]
if result_ids:
cache.store_search_results('itunes', 'album', query, limit, result_ids)
return result
def get_album(self, album_id: str, include_tracks: bool = True) -> Optional[Dict[str, Any]]:
"""Get album information with tracks - normalized to Spotify format.
@ -503,6 +589,54 @@ class iTunesClient:
album_id: iTunes album/collection ID
include_tracks: If True, also fetches and includes tracks (default True for Spotify compatibility)
"""
# Check cache for raw album data
cache = get_metadata_cache()
cached = cache.get_entity('itunes', 'album', str(album_id))
if cached:
# Reconstruct Spotify-compatible format from cached raw iTunes data
image_url = None
if cached.get('artworkUrl100'):
image_url = cached['artworkUrl100'].replace('100x100bb', '600x600bb')
images = []
if image_url:
images = [
{'url': image_url, 'height': 600, 'width': 600},
{'url': cached['artworkUrl100'].replace('100x100bb', '300x300bb'), 'height': 300, 'width': 300},
{'url': cached['artworkUrl100'], 'height': 100, 'width': 100}
]
track_count = cached.get('trackCount', 0)
if track_count <= 3:
album_type = 'single'
elif track_count <= 6:
album_type = 'ep'
else:
album_type = 'album'
album_result = {
'id': str(cached.get('collectionId', '')),
'name': _clean_itunes_album_name(cached.get('collectionName', '')),
'images': images,
'artists': [{'name': cached.get('artistName', 'Unknown Artist'), 'id': str(cached.get('artistId', ''))}],
'release_date': cached.get('releaseDate', '')[:10] if cached.get('releaseDate') else '',
'total_tracks': track_count,
'album_type': album_type,
'external_urls': {'itunes': cached.get('collectionViewUrl', '')},
'uri': f"itunes:album:{cached.get('collectionId', '')}",
'_source': 'itunes',
'_raw_data': cached
}
if include_tracks:
tracks_data = self.get_album_tracks(album_id)
if tracks_data and 'items' in tracks_data:
album_result['tracks'] = tracks_data
else:
album_result['tracks'] = {'items': [], 'total': 0}
return album_result
results = self._lookup(id=album_id)
for album_data in results:
@ -544,6 +678,9 @@ class iTunesClient:
'_raw_data': album_data
}
# Cache the raw album data
cache.store_entity('itunes', 'album', str(album_id), album_data)
# Include tracks to match Spotify's get_album format
if include_tracks:
tracks_data = self.get_album_tracks(album_id)
@ -558,6 +695,12 @@ class iTunesClient:
def get_album_tracks(self, album_id: str) -> Optional[Dict[str, Any]]:
"""Get album tracks - normalized to Spotify format"""
# Check cache for album tracks listing
cache = get_metadata_cache()
cached = cache.get_entity('itunes', 'album', f"{album_id}_tracks")
if cached:
return cached
results = self._lookup(id=album_id, entity='song')
if not results:
@ -629,12 +772,25 @@ class iTunesClient:
logger.info(f"Retrieved {len(tracks)} tracks for album {album_id}")
return {
result = {
'items': tracks,
'total': len(tracks),
'limit': len(tracks),
'next': None
}
# Cache the album tracks listing
cache.store_entity('itunes', 'album', f"{album_id}_tracks", result)
# Also cache individual tracks from the raw results
track_entries = []
for item in results:
if item.get('wrapperType') == 'track' and item.get('kind') == 'song' and item.get('trackId'):
track_entries.append((str(item['trackId']), item))
if track_entries:
cache.store_entities_bulk('itunes', 'track', track_entries)
return result
# ==================== Artist Methods ====================
@ -663,13 +819,35 @@ class iTunesClient:
Note: Artist images are not fetched during search to keep it fast.
Images are fetched when viewing artist details (get_artist method).
"""
# Check search cache
cache = get_metadata_cache()
cached_results = cache.get_search_results('itunes', 'artist', query, limit)
if cached_results is not None:
artists = []
for raw in cached_results:
try:
artists.append(Artist.from_itunes_artist(raw))
except Exception:
pass
if artists:
return artists
results = self._search(query, 'musicArtist', limit)
artists = []
raw_items = []
for artist_data in results:
if artist_data.get('wrapperType') == 'artist':
artist = Artist.from_itunes_artist(artist_data)
artists.append(artist)
raw_items.append(artist_data)
# Cache individual artists + search mapping
entries = [(str(ad.get('artistId', '')), ad) for ad in raw_items if ad.get('artistId')]
if entries:
cache.store_entities_bulk('itunes', 'artist', entries)
cache.store_search_results('itunes', 'artist', query, limit,
[str(ad.get('artistId', '')) for ad in raw_items if ad.get('artistId')])
return artists
@ -683,6 +861,37 @@ class iTunesClient:
Returns:
Dictionary with artist data matching Spotify's format
"""
# Check cache — stores raw iTunes data, reconstruct Spotify-compatible format
cache = get_metadata_cache()
cached = cache.get_entity('itunes', 'artist', str(artist_id))
if cached and cached.get('wrapperType') == 'artist':
# Reconstruct Spotify-compatible format from cached raw iTunes data
# Don't call _get_artist_image_from_albums here — avoid API call on cache hit
# The image_url was already extracted during initial caching
images = []
artwork_url = cached.get('artworkUrl100')
if artwork_url:
images = [
{'url': artwork_url.replace('100x100bb', '600x600bb'), 'height': 600, 'width': 600},
{'url': artwork_url.replace('100x100bb', '300x300bb'), 'height': 300, 'width': 300},
{'url': artwork_url, 'height': 100, 'width': 100}
]
genres = []
if cached.get('primaryGenreName'):
genres = [cached['primaryGenreName']]
return {
'id': str(cached.get('artistId', '')),
'name': cached.get('artistName', ''),
'images': images,
'genres': genres,
'popularity': 0,
'followers': {'total': 0},
'external_urls': {'itunes': cached.get('artistViewUrl', '')},
'uri': f"itunes:artist:{cached.get('artistId', '')}",
'_source': 'itunes',
'_raw_data': cached
}
results = self._lookup(id=artist_id)
for artist_data in results:
@ -698,6 +907,8 @@ class iTunesClient:
if album_art:
# Convert back to base URL format for building array
artwork_url = album_art.replace('600x600bb', '100x100bb')
# Store discovered artwork in raw data so cache hits will have it
artist_data['artworkUrl100'] = artwork_url
if artwork_url:
images = [
@ -711,7 +922,7 @@ class iTunesClient:
if artist_data.get('primaryGenreName'):
genres = [artist_data['primaryGenreName']]
return {
result = {
'id': str(artist_data.get('artistId', '')),
'name': artist_data.get('artistName', ''),
'images': images,
@ -723,6 +934,9 @@ class iTunesClient:
'_source': 'itunes',
'_raw_data': artist_data
}
# Cache the processed result (raw_data inside has original iTunes format)
cache.store_entity('itunes', 'artist', str(artist_id), artist_data)
return result
return None
@ -818,6 +1032,15 @@ class iTunesClient:
# Extract albums from dict
albums = [item['album'] for item in seen_albums.values()]
# Cache individual albums opportunistically
album_entries = []
for album_data in results:
if album_data.get('wrapperType') == 'collection' and album_data.get('collectionId'):
album_entries.append((str(album_data['collectionId']), album_data))
if album_entries:
cache = get_metadata_cache()
cache.store_entities_bulk('itunes', 'album', album_entries)
logger.info(f"Retrieved {len(albums)} unique albums for artist {artist_id} (filtered from {len(results)} results)")
return albums[:limit]

679
core/metadata_cache.py Normal file
View file

@ -0,0 +1,679 @@
"""
Universal Metadata Cache caches all Spotify and iTunes API responses.
Stores full JSON responses alongside structured queryable fields for browsing.
Transparent to callers: check cache before API call, store after success.
"""
import json
import logging
import threading
from datetime import datetime
from typing import Optional, Dict, List, Tuple
logger = logging.getLogger(__name__)
# Singleton
_cache_instance = None
_cache_lock = threading.Lock()
def get_metadata_cache():
"""Get or create the singleton MetadataCache instance."""
global _cache_instance
if _cache_instance is None:
with _cache_lock:
if _cache_instance is None:
_cache_instance = MetadataCache()
return _cache_instance
class MetadataCache:
"""Caches Spotify and iTunes API responses with structured fields + raw JSON."""
def __init__(self):
# Tables are created by MusicDatabase migration — we just use get_database()
pass
def _get_db(self):
from database.music_database import get_database
return get_database()
# ─── Entity Methods ───────────────────────────────────────────────
def get_entity(self, source: str, entity_type: str, entity_id: str) -> Optional[dict]:
"""Look up a cached entity. Returns parsed raw_json dict on hit, None on miss."""
try:
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
cursor.execute("""
SELECT id, raw_json FROM metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id = ?
""", (source, entity_type, entity_id))
row = cursor.fetchone()
if row:
# Touch: update access stats
cursor.execute("""
UPDATE metadata_cache_entities
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
WHERE id = ?
""", (row['id'],))
conn.commit()
conn.close()
return json.loads(row['raw_json'])
conn.close()
return None
except Exception as e:
logger.debug(f"Cache lookup error ({source}/{entity_type}/{entity_id}): {e}")
return None
def store_entity(self, source: str, entity_type: str, entity_id: str, raw_data: dict) -> None:
"""Store an entity in the cache. Extracts structured fields from raw_data."""
if not entity_id or not raw_data:
return
try:
fields = self._extract_fields(source, entity_type, raw_data)
raw_json = json.dumps(raw_data, default=str)
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
cursor.execute("""
INSERT OR REPLACE INTO metadata_cache_entities
(source, entity_type, entity_id, name, image_url, external_urls,
genres, popularity, followers,
artist_name, artist_id, release_date, total_tracks, album_type, label,
album_name, album_id, duration_ms, track_number, disc_number, explicit, isrc, preview_url,
raw_json, updated_at, last_accessed_at, access_count)
VALUES (?, ?, ?, ?, ?, ?,
?, ?, ?,
?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?,
?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP,
COALESCE((SELECT access_count FROM metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id = ?), 0) + 1)
""", (
source, entity_type, entity_id,
fields.get('name', ''),
fields.get('image_url'),
fields.get('external_urls'),
fields.get('genres'),
fields.get('popularity'),
fields.get('followers'),
fields.get('artist_name'),
fields.get('artist_id'),
fields.get('release_date'),
fields.get('total_tracks'),
fields.get('album_type'),
fields.get('label'),
fields.get('album_name'),
fields.get('album_id'),
fields.get('duration_ms'),
fields.get('track_number'),
fields.get('disc_number'),
fields.get('explicit'),
fields.get('isrc'),
fields.get('preview_url'),
raw_json,
source, entity_type, entity_id,
))
conn.commit()
conn.close()
except Exception as e:
logger.debug(f"Cache store error ({source}/{entity_type}/{entity_id}): {e}")
def store_entities_bulk(self, source: str, entity_type: str, items: List[Tuple[str, dict]]) -> None:
"""Store multiple entities at once. items = [(entity_id, raw_data), ...]"""
if not items:
return
try:
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
for entity_id, raw_data in items:
if not entity_id or not raw_data:
continue
fields = self._extract_fields(source, entity_type, raw_data)
raw_json = json.dumps(raw_data, default=str)
cursor.execute("""
INSERT OR REPLACE INTO metadata_cache_entities
(source, entity_type, entity_id, name, image_url, external_urls,
genres, popularity, followers,
artist_name, artist_id, release_date, total_tracks, album_type, label,
album_name, album_id, duration_ms, track_number, disc_number, explicit, isrc, preview_url,
raw_json, updated_at, last_accessed_at)
VALUES (?, ?, ?, ?, ?, ?,
?, ?, ?,
?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?,
?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
""", (
source, entity_type, entity_id,
fields.get('name', ''),
fields.get('image_url'),
fields.get('external_urls'),
fields.get('genres'),
fields.get('popularity'),
fields.get('followers'),
fields.get('artist_name'),
fields.get('artist_id'),
fields.get('release_date'),
fields.get('total_tracks'),
fields.get('album_type'),
fields.get('label'),
fields.get('album_name'),
fields.get('album_id'),
fields.get('duration_ms'),
fields.get('track_number'),
fields.get('disc_number'),
fields.get('explicit'),
fields.get('isrc'),
fields.get('preview_url'),
raw_json,
))
conn.commit()
conn.close()
except Exception as e:
logger.debug(f"Cache bulk store error ({source}/{entity_type}): {e}")
def get_entities_batch(self, source: str, entity_type: str,
entity_ids: List[str]) -> Tuple[Dict[str, dict], List[str]]:
"""Batch cache lookup. Returns (found_dict, missing_ids)."""
found = {}
missing = []
if not entity_ids:
return found, missing
try:
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
# Batch query in chunks of 500 to avoid SQLite variable limit
for i in range(0, len(entity_ids), 500):
chunk = entity_ids[i:i + 500]
placeholders = ','.join('?' * len(chunk))
cursor.execute(f"""
SELECT entity_id, raw_json FROM metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders})
""", [source, entity_type] + chunk)
for row in cursor.fetchall():
found[row['entity_id']] = json.loads(row['raw_json'])
# Touch all found entries
if found:
found_in_chunk = [eid for eid in chunk if eid in found]
if found_in_chunk:
ph2 = ','.join('?' * len(found_in_chunk))
cursor.execute(f"""
UPDATE metadata_cache_entities
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
WHERE source = ? AND entity_type = ? AND entity_id IN ({ph2})
""", [source, entity_type] + found_in_chunk)
conn.commit()
conn.close()
missing = [eid for eid in entity_ids if eid not in found]
except Exception as e:
logger.debug(f"Cache batch lookup error: {e}")
missing = entity_ids
return found, missing
# ─── Search Cache Methods ─────────────────────────────────────────
def get_search_results(self, source: str, search_type: str,
query: str, limit: int) -> Optional[List[dict]]:
"""Look up cached search results. Returns list of raw_json dicts or None."""
normalized = query.strip().lower()
if not normalized:
return None
try:
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
cursor.execute("""
SELECT id, result_ids, created_at FROM metadata_cache_searches
WHERE source = ? AND search_type = ? AND query_normalized = ? AND search_limit = ?
""", (source, search_type, normalized, limit))
row = cursor.fetchone()
if not row:
conn.close()
return None
# Check TTL (7 days for searches)
try:
created = datetime.fromisoformat(row['created_at'])
age_days = (datetime.now() - created).days
if age_days > 7:
# Expired — delete and return miss
cursor.execute("DELETE FROM metadata_cache_searches WHERE id = ?", (row['id'],))
conn.commit()
conn.close()
return None
except (ValueError, TypeError):
pass
# Touch search entry
cursor.execute("""
UPDATE metadata_cache_searches
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
WHERE id = ?
""", (row['id'],))
conn.commit()
# Resolve entity IDs to full data
result_ids = json.loads(row['result_ids'])
if not result_ids:
conn.close()
return []
results = []
for eid in result_ids:
cursor.execute("""
SELECT raw_json FROM metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id = ?
""", (source, search_type, eid))
erow = cursor.fetchone()
if erow:
results.append(json.loads(erow['raw_json']))
conn.close()
# Only return if we found all (or most) entries — partial results are unreliable
if len(results) >= len(result_ids) * 0.8:
return results
return None
except Exception as e:
logger.debug(f"Search cache lookup error ({source}/{search_type}/{query}): {e}")
return None
def store_search_results(self, source: str, search_type: str, query: str,
limit: int, entity_ids: List[str]) -> None:
"""Store search result mapping. Individual entities should already be stored."""
normalized = query.strip().lower()
if not normalized or not entity_ids:
return
try:
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
cursor.execute("""
INSERT OR REPLACE INTO metadata_cache_searches
(source, search_type, query_normalized, query_original, result_ids,
result_count, search_limit, last_accessed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
""", (
source, search_type, normalized, query.strip(),
json.dumps(entity_ids), len(entity_ids), limit,
))
conn.commit()
conn.close()
except Exception as e:
logger.debug(f"Search cache store error ({source}/{search_type}/{query}): {e}")
# ─── Browsing (for UI) ────────────────────────────────────────────
def browse(self, entity_type: str, source: str = None, search: str = None,
sort: str = 'last_accessed_at', sort_dir: str = 'desc',
offset: int = 0, limit: int = 48) -> dict:
"""Paginated browse of cached entities for the UI."""
try:
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
where_clauses = ['entity_type = ?']
params = [entity_type]
# Exclude pseudo-entities like album_id_tracks and track_id_features
where_clauses.append(r"entity_id NOT LIKE '%\_tracks' ESCAPE '\'")
where_clauses.append(r"entity_id NOT LIKE '%\_features' ESCAPE '\'")
if source:
where_clauses.append('source = ?')
params.append(source)
if search:
search_term = f'%{search}%'
where_clauses.append('(name LIKE ? OR artist_name LIKE ? OR album_name LIKE ?)')
params.extend([search_term, search_term, search_term])
where_sql = ' AND '.join(where_clauses)
# Count total
cursor.execute(f"SELECT COUNT(*) as cnt FROM metadata_cache_entities WHERE {where_sql}", params)
total = cursor.fetchone()['cnt']
# Validate sort column
valid_sorts = {'last_accessed_at', 'created_at', 'access_count', 'name', 'popularity', 'updated_at'}
if sort not in valid_sorts:
sort = 'last_accessed_at'
direction = 'ASC' if sort_dir == 'asc' else 'DESC'
cursor.execute(f"""
SELECT id, source, entity_type, entity_id, name, image_url,
genres, popularity, followers,
artist_name, artist_id, release_date, total_tracks, album_type, label,
album_name, album_id, duration_ms, track_number, disc_number, explicit,
isrc, preview_url, external_urls,
created_at, updated_at, last_accessed_at, access_count
FROM metadata_cache_entities
WHERE {where_sql}
ORDER BY {sort} {direction}
LIMIT ? OFFSET ?
""", params + [limit, offset])
items = []
for row in cursor.fetchall():
item = dict(row)
# Parse JSON fields for the UI
for json_field in ('genres', 'external_urls'):
if item.get(json_field):
try:
item[json_field] = json.loads(item[json_field])
except (json.JSONDecodeError, TypeError):
pass
items.append(item)
conn.close()
return {'items': items, 'total': total, 'offset': offset, 'limit': limit}
except Exception as e:
logger.error(f"Cache browse error: {e}")
return {'items': [], 'total': 0, 'offset': offset, 'limit': limit}
def get_entity_detail(self, source: str, entity_type: str, entity_id: str) -> Optional[dict]:
"""Get full entity detail including parsed raw_json for the detail modal."""
try:
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
cursor.execute("""
SELECT * FROM metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id = ?
""", (source, entity_type, entity_id))
row = cursor.fetchone()
if not row:
conn.close()
return None
# Touch
cursor.execute("""
UPDATE metadata_cache_entities
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
WHERE id = ?
""", (row['id'],))
conn.commit()
item = dict(row)
# Parse JSON fields
for json_field in ('genres', 'external_urls', 'raw_json'):
if item.get(json_field):
try:
item[json_field] = json.loads(item[json_field])
except (json.JSONDecodeError, TypeError):
pass
conn.close()
return item
except Exception as e:
logger.error(f"Cache detail error: {e}")
return None
# ─── Stats ────────────────────────────────────────────────────────
def get_stats(self) -> dict:
"""Get cache statistics for the dashboard tool card and modal stats bar."""
try:
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
stats = {
'artists': {'spotify': 0, 'itunes': 0},
'albums': {'spotify': 0, 'itunes': 0},
'tracks': {'spotify': 0, 'itunes': 0},
'searches': 0,
'total_entries': 0,
'total_hits': 0,
'oldest': None,
'newest': None,
}
# Count by type and source (exclude pseudo-entities like _tracks, _features)
cursor.execute(r"""
SELECT entity_type, source, COUNT(*) as cnt, SUM(access_count) as hits
FROM metadata_cache_entities
WHERE entity_id NOT LIKE '%\_tracks' ESCAPE '\'
AND entity_id NOT LIKE '%\_features' ESCAPE '\'
GROUP BY entity_type, source
""")
type_key_map = {'artist': 'artists', 'album': 'albums', 'track': 'tracks'}
for row in cursor.fetchall():
et = type_key_map.get(row['entity_type'])
src = row['source']
if et and et in stats and src in stats[et]:
stats[et][src] = row['cnt']
stats['total_entries'] += row['cnt']
stats['total_hits'] += (row['hits'] or 0)
# Search count
cursor.execute("SELECT COUNT(*) as cnt FROM metadata_cache_searches")
stats['searches'] = cursor.fetchone()['cnt']
# Oldest and newest
cursor.execute("SELECT MIN(created_at) as oldest, MAX(created_at) as newest FROM metadata_cache_entities")
row = cursor.fetchone()
if row:
stats['oldest'] = row['oldest']
stats['newest'] = row['newest']
conn.close()
return stats
except Exception as e:
logger.error(f"Cache stats error: {e}")
return {
'artists': {'spotify': 0, 'itunes': 0},
'albums': {'spotify': 0, 'itunes': 0},
'tracks': {'spotify': 0, 'itunes': 0},
'searches': 0, 'total_entries': 0, 'total_hits': 0,
'oldest': None, 'newest': None,
}
# ─── Maintenance ──────────────────────────────────────────────────
def evict_expired(self) -> int:
"""Delete entries that have exceeded their TTL. Returns count of evicted entries."""
try:
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
# Entities
cursor.execute("""
DELETE FROM metadata_cache_entities
WHERE julianday('now') - julianday(updated_at) > ttl_days
""")
entity_count = cursor.rowcount
# Searches
cursor.execute("""
DELETE FROM metadata_cache_searches
WHERE julianday('now') - julianday(created_at) > ttl_days
""")
search_count = cursor.rowcount
conn.commit()
conn.close()
total = entity_count + search_count
if total > 0:
logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)")
return total
except Exception as e:
logger.error(f"Cache eviction error: {e}")
return 0
def clear(self, source: str = None, entity_type: str = None) -> int:
"""Clear cache entries. Optional filters by source and/or entity_type."""
try:
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
# Clear entities
where_parts = []
params = []
if source:
where_parts.append('source = ?')
params.append(source)
if entity_type:
where_parts.append('entity_type = ?')
params.append(entity_type)
if where_parts:
where_sql = ' AND '.join(where_parts)
cursor.execute(f"DELETE FROM metadata_cache_entities WHERE {where_sql}", params)
else:
cursor.execute("DELETE FROM metadata_cache_entities")
entity_count = cursor.rowcount
# Clear searches (match source and entity_type → search_type)
search_where = []
search_params = []
if source:
search_where.append('source = ?')
search_params.append(source)
if entity_type:
search_where.append('search_type = ?')
search_params.append(entity_type)
if search_where:
cursor.execute(f"DELETE FROM metadata_cache_searches WHERE {' AND '.join(search_where)}", search_params)
else:
cursor.execute("DELETE FROM metadata_cache_searches")
search_count = cursor.rowcount
conn.commit()
conn.close()
total = entity_count + search_count
logger.info(f"Cleared {total} cache entries (source={source}, type={entity_type})")
return total
except Exception as e:
logger.error(f"Cache clear error: {e}")
return 0
# ─── Field Extraction ─────────────────────────────────────────────
def _extract_fields(self, source: str, entity_type: str, raw_data: dict) -> dict:
"""Extract structured queryable fields from a raw API response."""
if source == 'spotify':
return self._extract_spotify_fields(entity_type, raw_data)
elif source == 'itunes':
return self._extract_itunes_fields(entity_type, raw_data)
return {'name': str(raw_data.get('name', raw_data.get('trackName', '')))}
def _extract_spotify_fields(self, entity_type: str, data: dict) -> dict:
"""Extract fields from Spotify API response."""
fields = {}
if entity_type == 'artist':
fields['name'] = data.get('name', '')
fields['genres'] = json.dumps(data.get('genres', []))
fields['popularity'] = data.get('popularity', 0)
followers = data.get('followers')
fields['followers'] = followers.get('total', 0) if isinstance(followers, dict) else 0
images = data.get('images', [])
fields['image_url'] = images[0]['url'] if images else None
fields['external_urls'] = json.dumps(data.get('external_urls', {}))
elif entity_type == 'album':
fields['name'] = data.get('name', '')
artists = data.get('artists', [])
if artists:
fields['artist_name'] = artists[0].get('name', '')
fields['artist_id'] = artists[0].get('id', '')
fields['release_date'] = data.get('release_date', '')
fields['total_tracks'] = data.get('total_tracks', 0)
fields['album_type'] = data.get('album_type', 'album')
fields['label'] = data.get('label', '')
images = data.get('images', [])
fields['image_url'] = images[0]['url'] if images else None
fields['genres'] = json.dumps(data.get('genres', []))
fields['external_urls'] = json.dumps(data.get('external_urls', {}))
elif entity_type == 'track':
fields['name'] = data.get('name', '')
artists = data.get('artists', [])
if artists:
fields['artist_name'] = artists[0].get('name', '')
fields['artist_id'] = artists[0].get('id', '')
album = data.get('album', {})
fields['album_name'] = album.get('name', '')
fields['album_id'] = album.get('id', '')
album_images = album.get('images', [])
fields['image_url'] = album_images[0]['url'] if album_images else None
fields['duration_ms'] = data.get('duration_ms', 0)
fields['track_number'] = data.get('track_number')
fields['disc_number'] = data.get('disc_number', 1)
fields['explicit'] = 1 if data.get('explicit') else 0
fields['popularity'] = data.get('popularity', 0)
ext_ids = data.get('external_ids', {})
fields['isrc'] = ext_ids.get('isrc') if isinstance(ext_ids, dict) else None
fields['preview_url'] = data.get('preview_url')
fields['external_urls'] = json.dumps(data.get('external_urls', {}))
return fields
def _extract_itunes_fields(self, entity_type: str, data: dict) -> dict:
"""Extract fields from iTunes API response."""
fields = {}
def _upscale_artwork(url):
"""Convert iTunes 100x100 artwork to 600x600."""
if url and '100x100' in url:
return url.replace('100x100', '600x600')
return url
if entity_type == 'artist':
fields['name'] = data.get('artistName', '')
genre = data.get('primaryGenreName', '')
fields['genres'] = json.dumps([genre] if genre else [])
fields['popularity'] = 0
fields['followers'] = 0
fields['image_url'] = _upscale_artwork(data.get('artworkUrl100'))
urls = {}
if data.get('artistViewUrl'):
urls['itunes'] = data['artistViewUrl']
fields['external_urls'] = json.dumps(urls)
elif entity_type == 'album':
fields['name'] = data.get('collectionName', '')
fields['artist_name'] = data.get('artistName', '')
fields['artist_id'] = str(data.get('artistId', ''))
fields['release_date'] = data.get('releaseDate', '')[:10] if data.get('releaseDate') else ''
fields['total_tracks'] = data.get('trackCount', 0)
# Infer album type from track count
tc = data.get('trackCount', 0)
if tc <= 3:
fields['album_type'] = 'single'
elif tc <= 6:
fields['album_type'] = 'ep'
else:
fields['album_type'] = 'album'
fields['image_url'] = _upscale_artwork(data.get('artworkUrl100'))
urls = {}
if data.get('collectionViewUrl'):
urls['itunes'] = data['collectionViewUrl']
fields['external_urls'] = json.dumps(urls)
elif entity_type == 'track':
fields['name'] = data.get('trackName', '')
fields['artist_name'] = data.get('artistName', '')
fields['artist_id'] = str(data.get('artistId', ''))
fields['album_name'] = data.get('collectionName', '')
fields['album_id'] = str(data.get('collectionId', ''))
fields['image_url'] = _upscale_artwork(data.get('artworkUrl100'))
fields['duration_ms'] = data.get('trackTimeMillis', 0)
fields['track_number'] = data.get('trackNumber')
fields['disc_number'] = data.get('discNumber', 1)
fields['explicit'] = 1 if data.get('trackExplicitness') == 'explicit' else 0
fields['preview_url'] = data.get('previewUrl')
urls = {}
if data.get('trackViewUrl'):
urls['itunes'] = data['trackViewUrl']
fields['external_urls'] = json.dumps(urls)
return fields

View file

@ -7,6 +7,7 @@ from functools import wraps
from dataclasses import dataclass
from utils.logging_config import get_logger
from config.settings import config_manager
from core.metadata_cache import get_metadata_cache
logger = get_logger("spotify_client")
@ -920,15 +921,39 @@ class SpotifyClient:
@rate_limited
def search_tracks(self, query: str, limit: int = 10) -> List[Track]:
"""Search for tracks - falls back to iTunes if Spotify not authenticated"""
if self.is_spotify_authenticated():
cache = get_metadata_cache()
use_spotify = self.is_spotify_authenticated()
source = 'spotify' if use_spotify else 'itunes'
# Check search cache
cached_results = cache.get_search_results(source, 'track', query, min(limit, 10))
if cached_results is not None:
tracks = []
for raw in cached_results:
try:
tracks.append(Track.from_spotify_track(raw) if source == 'spotify' else Track.from_itunes_track(raw))
except Exception:
pass
if tracks:
return tracks
if use_spotify:
try:
results = self.sp.search(q=query, type='track', limit=min(limit, 10))
tracks = []
raw_items = results['tracks']['items']
for track_data in results['tracks']['items']:
for track_data in raw_items:
track = Track.from_spotify_track(track_data)
tracks.append(track)
# Cache individual tracks + search mapping
entries = [(td.get('id'), td) for td in raw_items if td.get('id')]
if entries:
cache.store_entities_bulk('spotify', 'track', entries)
cache.store_search_results('spotify', 'track', query, min(limit, 10),
[td.get('id') for td in raw_items if td.get('id')])
return tracks
except Exception as e:
@ -942,16 +967,42 @@ class SpotifyClient:
@rate_limited
def search_artists(self, query: str, limit: int = 10) -> List[Artist]:
"""Search for artists - falls back to iTunes if Spotify not authenticated"""
if self.is_spotify_authenticated():
cache = get_metadata_cache()
use_spotify = self.is_spotify_authenticated()
source = 'spotify' if use_spotify else 'itunes'
# Check search cache
cached_results = cache.get_search_results(source, 'artist', query, min(limit, 10))
if cached_results is not None:
artists = []
for raw in cached_results:
try:
artists.append(Artist.from_spotify_artist(raw) if source == 'spotify' else Artist.from_itunes_artist(raw))
except Exception:
pass
if artists:
query_lower = query.lower().strip()
artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1))
return artists
if use_spotify:
try:
search_query = f'artist:{query}' if len(query.strip()) <= 4 else query
results = self.sp.search(q=search_query, type='artist', limit=min(limit, 10))
artists = []
raw_items = results['artists']['items']
for artist_data in results['artists']['items']:
for artist_data in raw_items:
artist = Artist.from_spotify_artist(artist_data)
artists.append(artist)
# Cache individual artists + search mapping
entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')]
if entries:
cache.store_entities_bulk('spotify', 'artist', entries)
cache.store_search_results('spotify', 'artist', query, min(limit, 10),
[ad.get('id') for ad in raw_items if ad.get('id')])
# Re-rank: boost exact name matches to the top
query_lower = query.lower().strip()
artists.sort(key=lambda a: (0 if a.name.lower().strip() == query_lower else 1))
@ -972,15 +1023,39 @@ class SpotifyClient:
@rate_limited
def search_albums(self, query: str, limit: int = 10) -> List[Album]:
"""Search for albums - falls back to iTunes if Spotify not authenticated"""
if self.is_spotify_authenticated():
cache = get_metadata_cache()
use_spotify = self.is_spotify_authenticated()
source = 'spotify' if use_spotify else 'itunes'
# Check search cache
cached_results = cache.get_search_results(source, 'album', query, min(limit, 10))
if cached_results is not None:
albums = []
for raw in cached_results:
try:
albums.append(Album.from_spotify_album(raw) if source == 'spotify' else Album.from_itunes_album(raw))
except Exception:
pass
if albums:
return albums
if use_spotify:
try:
results = self.sp.search(q=query, type='album', limit=min(limit, 10))
albums = []
raw_items = results['albums']['items']
for album_data in results['albums']['items']:
for album_data in raw_items:
album = Album.from_spotify_album(album_data)
albums.append(album)
# Cache individual albums + search mapping
entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')]
if entries:
cache.store_entities_bulk('spotify', 'album', entries)
cache.store_search_results('spotify', 'album', query, min(limit, 10),
[ad.get('id') for ad in raw_items if ad.get('id')])
return albums
except Exception as e:
@ -994,33 +1069,25 @@ class SpotifyClient:
@rate_limited
def get_track_details(self, track_id: str) -> Optional[Dict[str, Any]]:
"""Get detailed track information - falls back to iTunes if Spotify not authenticated"""
# Check cache — we store raw track_data, reconstruct enhanced on hit
cache = get_metadata_cache()
source = 'itunes' if self._is_itunes_id(track_id) else 'spotify'
cached = cache.get_entity(source, 'track', track_id)
if cached:
if source == 'spotify':
return self._build_enhanced_track(cached)
# iTunes cache hit — delegate to iTunesClient which reconstructs enhanced format
return self._itunes.get_track_details(track_id)
if self.is_spotify_authenticated():
try:
track_data = self.sp.track(track_id)
# Enhance with additional useful metadata for our purposes
if track_data:
enhanced_data = {
'id': track_data['id'],
'name': track_data['name'],
'track_number': track_data['track_number'],
'disc_number': track_data['disc_number'],
'duration_ms': track_data['duration_ms'],
'explicit': track_data['explicit'],
'artists': [artist['name'] for artist in track_data['artists']],
'primary_artist': track_data['artists'][0]['name'] if track_data['artists'] else None,
'album': {
'id': track_data['album']['id'],
'name': track_data['album']['name'],
'total_tracks': track_data['album']['total_tracks'],
'release_date': track_data['album']['release_date'],
'album_type': track_data['album']['album_type'],
'artists': [artist['name'] for artist in track_data['album']['artists']]
},
'is_album_track': track_data['album']['total_tracks'] > 1,
'raw_data': track_data # Keep original for fallback
}
return enhanced_data
# Cache the raw Spotify response
cache.store_entity('spotify', 'track', track_id, track_data)
return self._build_enhanced_track(track_data)
return track_data
except Exception as e:
@ -1030,20 +1097,55 @@ class SpotifyClient:
# iTunes fallback - only if ID is numeric (iTunes format)
if self._is_itunes_id(track_id):
logger.debug(f"Using iTunes fallback for track details: {track_id}")
return self._itunes.get_track_details(track_id)
result = self._itunes.get_track_details(track_id)
return result
else:
logger.debug(f"Cannot use iTunes fallback for Spotify track ID: {track_id}")
return None
@staticmethod
def _build_enhanced_track(track_data: dict) -> dict:
"""Build enhanced track dict from raw Spotify track data."""
return {
'id': track_data['id'],
'name': track_data['name'],
'track_number': track_data['track_number'],
'disc_number': track_data['disc_number'],
'duration_ms': track_data['duration_ms'],
'explicit': track_data['explicit'],
'artists': [artist['name'] for artist in track_data['artists']],
'primary_artist': track_data['artists'][0]['name'] if track_data['artists'] else None,
'album': {
'id': track_data['album']['id'],
'name': track_data['album']['name'],
'total_tracks': track_data['album']['total_tracks'],
'release_date': track_data['album']['release_date'],
'album_type': track_data['album']['album_type'],
'artists': [artist['name'] for artist in track_data['album']['artists']]
},
'is_album_track': track_data['album']['total_tracks'] > 1,
'raw_data': track_data
}
@rate_limited
def get_track_features(self, track_id: str) -> Optional[Dict[str, Any]]:
# Check cache — use entity_id with '_features' suffix
cache = get_metadata_cache()
cache_key = f"{track_id}_features"
cached = cache.get_entity('spotify', 'track', cache_key)
if cached:
return cached
if not self.is_spotify_authenticated():
return None
try:
features = self.sp.audio_features(track_id)
return features[0] if features else None
result = features[0] if features else None
if result:
cache.store_entity('spotify', 'track', cache_key, result)
return result
except Exception as e:
logger.error(f"Error fetching track features: {e}")
return None
@ -1051,9 +1153,21 @@ class SpotifyClient:
@rate_limited
def get_album(self, album_id: str) -> Optional[Dict[str, Any]]:
"""Get album information - falls back to iTunes if Spotify not authenticated"""
# Check cache first
cache = get_metadata_cache()
source = 'itunes' if self._is_itunes_id(album_id) else 'spotify'
cached = cache.get_entity(source, 'album', album_id)
if cached:
if source == 'spotify':
return cached # Spotify raw format is the expected format
# iTunes cache hit — delegate to iTunesClient which reconstructs Spotify-compatible format
return self._itunes.get_album(album_id)
if self.is_spotify_authenticated():
try:
album_data = self.sp.album(album_id)
if album_data:
cache.store_entity('spotify', 'album', album_id, album_data)
return album_data
except Exception as e:
@ -1064,7 +1178,8 @@ class SpotifyClient:
# iTunes fallback - only if ID is numeric (iTunes format)
if self._is_itunes_id(album_id):
logger.debug(f"Using iTunes fallback for album: {album_id}")
return self._itunes.get_album(album_id)
result = self._itunes.get_album(album_id)
return result
else:
logger.debug(f"Cannot use iTunes fallback for Spotify album ID: {album_id}")
return None
@ -1072,6 +1187,14 @@ class SpotifyClient:
@rate_limited
def get_album_tracks(self, album_id: str) -> Optional[Dict[str, Any]]:
"""Get album tracks - falls back to iTunes if Spotify not authenticated"""
# Cache key uses album_id with '_tracks' suffix to differentiate from album metadata
cache = get_metadata_cache()
source = 'itunes' if self._is_itunes_id(album_id) else 'spotify'
cache_key = f"{album_id}_tracks"
cached = cache.get_entity(source, 'album', cache_key)
if cached:
return cached
if self.is_spotify_authenticated():
try:
# Get first page of tracks
@ -1098,6 +1221,18 @@ class SpotifyClient:
result['next'] = None # No more pages
result['limit'] = len(all_tracks) # Update to reflect all tracks fetched
# Cache the aggregated result
cache.store_entity('spotify', 'album', cache_key, result)
# Also cache individual tracks opportunistically
track_entries = []
for track in all_tracks:
tid = track.get('id')
if tid:
track_entries.append((tid, track))
if track_entries:
cache.store_entities_bulk('spotify', 'track', track_entries)
return result
except Exception as e:
@ -1107,7 +1242,8 @@ class SpotifyClient:
# iTunes fallback - only if ID is numeric (iTunes format)
if self._is_itunes_id(album_id):
logger.debug(f"Using iTunes fallback for album tracks: {album_id}")
return self._itunes.get_album_tracks(album_id)
result = self._itunes.get_album_tracks(album_id)
return result
else:
logger.debug(f"Cannot use iTunes fallback for Spotify album ID: {album_id}")
return None
@ -1118,17 +1254,26 @@ class SpotifyClient:
if self.is_spotify_authenticated():
try:
albums = []
raw_items = []
results = self.sp.artist_albums(artist_id, album_type=album_type, limit=min(limit, 10))
while results:
for album_data in results['items']:
album = Album.from_spotify_album(album_data)
albums.append(album)
raw_items.append(album_data)
# Get next batch if available
results = self.sp.next(results) if results['next'] else None
logger.info(f"Retrieved {len(albums)} albums for artist {artist_id}")
# Cache individual albums opportunistically
cache = get_metadata_cache()
entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')]
if entries:
cache.store_entities_bulk('spotify', 'album', entries)
return albums
except Exception as e:
@ -1165,9 +1310,22 @@ class SpotifyClient:
Returns:
Dictionary with artist data including images, genres, popularity
"""
# Check cache first (works even during rate limit bans)
cache = get_metadata_cache()
source = 'itunes' if self._is_itunes_id(artist_id) else 'spotify'
cached = cache.get_entity(source, 'artist', artist_id)
if cached:
if source == 'spotify':
return cached # Spotify raw format is the expected format
# iTunes cache hit — delegate to iTunesClient which reconstructs Spotify-compatible format
return self._itunes.get_artist(artist_id)
if self.is_spotify_authenticated():
try:
return self.sp.artist(artist_id)
result = self.sp.artist(artist_id)
if result:
cache.store_entity('spotify', 'artist', artist_id, result)
return result
except Exception as e:
_detect_and_set_rate_limit(e, 'get_artist')
logger.error(f"Error fetching artist via Spotify: {e}")
@ -1176,7 +1334,34 @@ class SpotifyClient:
# iTunes fallback - only if ID is numeric (iTunes format)
if self._is_itunes_id(artist_id):
logger.debug(f"Using iTunes fallback for artist: {artist_id}")
return self._itunes.get_artist(artist_id)
result = self._itunes.get_artist(artist_id)
return result
else:
logger.debug(f"Cannot use iTunes fallback for Spotify artist ID: {artist_id}")
return None
return None
@rate_limited
def get_artists_batch(self, artist_ids: List[str]) -> Dict[str, Dict]:
"""Get multiple artists, using cache where possible, batch API for misses.
Returns dict keyed by artist_id artist data dict."""
if not artist_ids:
return {}
cache = get_metadata_cache()
found, missing = cache.get_entities_batch('spotify', 'artist', artist_ids)
if missing and self.is_spotify_authenticated():
try:
# Spotify batch endpoint accepts up to 50 IDs
for i in range(0, len(missing), 50):
chunk = missing[i:i + 50]
batch_result = self.sp.artists(chunk)
for artist_data in (batch_result or {}).get('artists', []):
if artist_data and artist_data.get('id'):
aid = artist_data['id']
cache.store_entity('spotify', 'artist', aid, artist_data)
found[aid] = artist_data
except Exception as e:
logger.error(f"Error in batch artist fetch: {e}")
return found

View file

@ -352,6 +352,9 @@ class MusicDatabase:
# Spotify library cache
self._add_spotify_library_cache_table(cursor)
# Universal metadata cache (Spotify + iTunes API responses)
self._add_metadata_cache_tables(cursor)
# Mirrored playlists — persistent backup of parsed playlists from any service
cursor.execute("""
CREATE TABLE IF NOT EXISTS mirrored_playlists (
@ -2296,6 +2299,84 @@ class MusicDatabase:
except Exception as e:
logger.error(f"Error creating spotify_library_cache table: {e}")
def _add_metadata_cache_tables(self, cursor):
"""Create metadata_cache_entities and metadata_cache_searches tables for universal API response caching"""
try:
cursor.execute("SELECT value FROM metadata WHERE key = 'metadata_cache_v1' LIMIT 1")
if cursor.fetchone():
return # Already migrated
logger.info("Creating metadata cache tables...")
cursor.execute("""
CREATE TABLE IF NOT EXISTS metadata_cache_entities (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL,
entity_type TEXT NOT NULL,
entity_id TEXT NOT NULL,
name TEXT NOT NULL,
image_url TEXT,
external_urls TEXT,
genres TEXT,
popularity INTEGER,
followers INTEGER,
artist_name TEXT,
artist_id TEXT,
release_date TEXT,
total_tracks INTEGER,
album_type TEXT,
label TEXT,
album_name TEXT,
album_id TEXT,
duration_ms INTEGER,
track_number INTEGER,
disc_number INTEGER,
explicit INTEGER,
isrc TEXT,
preview_url TEXT,
raw_json TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_accessed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
access_count INTEGER DEFAULT 1,
ttl_days INTEGER DEFAULT 30,
UNIQUE(source, entity_type, entity_id)
)
""")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_mce_lookup ON metadata_cache_entities (source, entity_type, entity_id)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_mce_name ON metadata_cache_entities (entity_type, name COLLATE NOCASE)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_mce_artist ON metadata_cache_entities (artist_name COLLATE NOCASE)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_mce_accessed ON metadata_cache_entities (last_accessed_at)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_mce_source ON metadata_cache_entities (source)")
cursor.execute("""
CREATE TABLE IF NOT EXISTS metadata_cache_searches (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL,
search_type TEXT NOT NULL,
query_normalized TEXT NOT NULL,
query_original TEXT NOT NULL,
result_ids TEXT NOT NULL,
result_count INTEGER NOT NULL,
search_limit INTEGER,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_accessed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
access_count INTEGER DEFAULT 1,
ttl_days INTEGER DEFAULT 7,
UNIQUE(source, search_type, query_normalized, search_limit)
)
""")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_mcs_lookup ON metadata_cache_searches (source, search_type, query_normalized)")
cursor.execute("INSERT OR REPLACE INTO metadata (key, value) VALUES ('metadata_cache_v1', '1')")
logger.info("Metadata cache tables created successfully")
except Exception as e:
logger.error(f"Error creating metadata cache tables: {e}")
# ── Profile CRUD ──────────────────────────────────────────────────
def get_all_profiles(self) -> List[Dict[str, Any]]:

View file

@ -67,6 +67,7 @@ from core.matching_engine import MusicMatchingEngine
from core.database_update_worker import DatabaseUpdateWorker, DatabaseStatsWorker
from core.web_scan_manager import WebScanManager
from core.lyrics_client import lyrics_client
from core.metadata_cache import get_metadata_cache
from database.music_database import get_database, MusicDatabase
from services.sync_service import PlaylistSyncService
@ -18350,6 +18351,88 @@ def download_backup_endpoint(filename):
except Exception as e:
return jsonify({"success": False, "error": str(e)}), 500
# ===============================
# == METADATA CACHE API ==
# ===============================
@app.route('/api/metadata-cache/stats', methods=['GET'])
def metadata_cache_stats():
"""Get metadata cache statistics."""
try:
cache = get_metadata_cache()
stats = cache.get_stats()
return jsonify(stats)
except Exception as e:
logger.error(f"Error getting metadata cache stats: {e}")
return jsonify({"error": str(e)}), 500
@app.route('/api/metadata-cache/browse', methods=['GET'])
def metadata_cache_browse():
"""Browse cached metadata entities with filtering, search, sorting, and pagination."""
try:
cache = get_metadata_cache()
entity_type = request.args.get('type', 'artist')
source = request.args.get('source')
search = request.args.get('search')
sort = request.args.get('sort', 'last_accessed_at')
sort_dir = request.args.get('sort_dir', 'desc')
offset = int(request.args.get('offset', 0))
limit = int(request.args.get('limit', 48))
result = cache.browse(
entity_type=entity_type,
source=source if source else None,
search=search if search else None,
sort=sort,
sort_dir=sort_dir,
offset=offset,
limit=limit
)
return jsonify(result)
except Exception as e:
logger.error(f"Error browsing metadata cache: {e}")
return jsonify({"error": str(e)}), 500
@app.route('/api/metadata-cache/entity/<source>/<entity_type>/<entity_id>', methods=['GET'])
def metadata_cache_entity_detail(source, entity_type, entity_id):
"""Get detailed view of a single cached entity."""
try:
cache = get_metadata_cache()
detail = cache.get_entity_detail(source, entity_type, entity_id)
if detail is None:
return jsonify({"error": "Entity not found"}), 404
return jsonify(detail)
except Exception as e:
logger.error(f"Error getting metadata cache entity: {e}")
return jsonify({"error": str(e)}), 500
@app.route('/api/metadata-cache/clear', methods=['DELETE'])
def metadata_cache_clear():
"""Clear cached metadata. Optional query params: source, type."""
try:
cache = get_metadata_cache()
source = request.args.get('source')
entity_type = request.args.get('type')
cleared = cache.clear(
source=source if source else None,
entity_type=entity_type if entity_type else None
)
return jsonify({"success": True, "cleared": cleared})
except Exception as e:
logger.error(f"Error clearing metadata cache: {e}")
return jsonify({"success": False, "error": str(e)}), 500
@app.route('/api/metadata-cache/evict', methods=['POST'])
def metadata_cache_evict():
"""Evict expired entries from the metadata cache."""
try:
cache = get_metadata_cache()
evicted = cache.evict_expired()
return jsonify({"success": True, "evicted": evicted})
except Exception as e:
logger.error(f"Error evicting metadata cache: {e}")
return jsonify({"success": False, "error": str(e)}), 500
# ===============================
# == QUALITY SCANNER ==
# ===============================

View file

@ -902,6 +902,37 @@
</div>
<div id="backup-list-container" class="backup-list-container"></div>
</div>
<!-- Metadata Cache Tool Card -->
<div class="tool-card" id="metadata-cache-card">
<div class="tool-card-header">
<h4 class="tool-card-title">Metadata Cache</h4>
<button class="tool-help-button" data-tool="metadata-cache"
title="Learn more about this tool">?</button>
</div>
<p class="tool-card-info">Cached API responses from Spotify &amp; iTunes</p>
<div class="tool-card-stats">
<div class="stat-item">
<span class="stat-item-label">Artists:</span>
<span class="stat-item-value" id="mcache-stat-artists">0</span>
</div>
<div class="stat-item">
<span class="stat-item-label">Albums:</span>
<span class="stat-item-value" id="mcache-stat-albums">0</span>
</div>
<div class="stat-item">
<span class="stat-item-label">Tracks:</span>
<span class="stat-item-value" id="mcache-stat-tracks">0</span>
</div>
<div class="stat-item">
<span class="stat-item-label">Hits:</span>
<span class="stat-item-value" id="mcache-stat-hits">0</span>
</div>
</div>
<div class="tool-card-controls">
<button id="mcache-browse-button" onclick="openMetadataCacheModal()">Browse Cache</button>
</div>
</div>
</div>
</div>
@ -5420,6 +5451,74 @@
</div>
</div>
<!-- Metadata Cache Browse Modal -->
<div class="mcache-modal-overlay" id="mcache-browse-modal" style="display:none;">
<div class="mcache-modal">
<div class="mcache-modal-header">
<h3 class="mcache-modal-title">Metadata Cache Browser</h3>
<div class="mcache-modal-header-actions">
<button class="mcache-btn-clear" onclick="clearMetadataCache()" title="Clear all cached data">Clear</button>
<button class="mcache-modal-close" onclick="closeMetadataCacheModal()">&times;</button>
</div>
</div>
<div class="mcache-stats-bar" id="mcache-stats-bar">
<div class="mcache-stat-pill">
<span class="mcache-stat-pill-label">Spotify</span>
<span class="mcache-stat-pill-value" id="mcache-browse-spotify-count">0</span>
</div>
<div class="mcache-stat-pill">
<span class="mcache-stat-pill-label">iTunes</span>
<span class="mcache-stat-pill-value" id="mcache-browse-itunes-count">0</span>
</div>
<div class="mcache-stat-pill">
<span class="mcache-stat-pill-label">Total Hits</span>
<span class="mcache-stat-pill-value" id="mcache-browse-hits">0</span>
</div>
<div class="mcache-stat-pill">
<span class="mcache-stat-pill-label">Searches</span>
<span class="mcache-stat-pill-value" id="mcache-browse-searches">0</span>
</div>
</div>
<div class="mcache-tabs">
<button class="mcache-tab active" data-tab="artist" onclick="switchMetadataCacheTab('artist')">Artists</button>
<button class="mcache-tab" data-tab="album" onclick="switchMetadataCacheTab('album')">Albums</button>
<button class="mcache-tab" data-tab="track" onclick="switchMetadataCacheTab('track')">Tracks</button>
</div>
<div class="mcache-filters">
<input type="text" class="mcache-search-input" id="mcache-search" placeholder="Search cached metadata..." oninput="debouncedMetadataCacheSearch()">
<select class="mcache-source-filter" id="mcache-source-filter" onchange="loadMetadataCacheBrowse()">
<option value="">All Sources</option>
<option value="spotify">Spotify</option>
<option value="itunes">iTunes</option>
</select>
<select class="mcache-sort-filter" id="mcache-sort-filter" onchange="loadMetadataCacheBrowse()">
<option value="last_accessed_at">Recently Accessed</option>
<option value="updated_at">Recently Updated</option>
<option value="created_at">Recently Added</option>
<option value="access_count">Most Accessed</option>
<option value="name">Name (A-Z)</option>
</select>
</div>
<div class="mcache-grid" id="mcache-grid">
<!-- Cards populated by JS -->
</div>
<div class="mcache-pagination" id="mcache-pagination"></div>
</div>
</div>
<!-- Metadata Cache Detail Modal (overlays browse modal) -->
<div class="mcache-detail-overlay" id="mcache-detail-modal" style="display:none;">
<div class="mcache-detail-modal">
<div class="mcache-detail-header">
<h3 class="mcache-detail-title" id="mcache-detail-title">Entity Detail</h3>
<button class="mcache-modal-close" onclick="closeMetadataCacheDetail()">&times;</button>
</div>
<div class="mcache-detail-body" id="mcache-detail-body">
<!-- Detail content populated by JS -->
</div>
</div>
</div>
<!-- Right Sidebar Download Indicator (Global - outside all containers) -->
<div class="discover-download-sidebar" id="discover-download-sidebar">
<div class="discover-download-sidebar-header">

View file

@ -17489,6 +17489,384 @@ async function deleteBackup(filename) {
}
}
// ============================================
// == METADATA CACHE ==
// ============================================
async function loadMetadataCacheStats() {
try {
const response = await fetch('/api/metadata-cache/stats');
if (!response.ok) return;
const stats = await response.json();
const artistsEl = document.getElementById('mcache-stat-artists');
const albumsEl = document.getElementById('mcache-stat-albums');
const tracksEl = document.getElementById('mcache-stat-tracks');
const hitsEl = document.getElementById('mcache-stat-hits');
if (artistsEl) artistsEl.textContent = (stats.artists?.spotify || 0) + (stats.artists?.itunes || 0);
if (albumsEl) albumsEl.textContent = (stats.albums?.spotify || 0) + (stats.albums?.itunes || 0);
if (tracksEl) tracksEl.textContent = (stats.tracks?.spotify || 0) + (stats.tracks?.itunes || 0);
if (hitsEl) hitsEl.textContent = stats.total_hits || 0;
} catch (e) {
// Silently fail — cache may not be initialized yet
}
}
let _mcacheCurrentTab = 'artist';
let _mcachePage = 0;
let _mcacheSearchTimeout = null;
const MCACHE_PAGE_SIZE = 48;
function openMetadataCacheModal() {
const modal = document.getElementById('mcache-browse-modal');
if (modal) {
modal.style.display = 'flex';
_mcacheCurrentTab = 'artist';
_mcachePage = 0;
// Reset UI
document.querySelectorAll('.mcache-tab').forEach(t => t.classList.remove('active'));
document.querySelector('.mcache-tab[data-tab="artist"]')?.classList.add('active');
const searchInput = document.getElementById('mcache-search');
if (searchInput) searchInput.value = '';
const sourceFilter = document.getElementById('mcache-source-filter');
if (sourceFilter) sourceFilter.value = '';
const sortFilter = document.getElementById('mcache-sort-filter');
if (sortFilter) sortFilter.value = 'last_accessed_at';
loadMetadataCacheBrowseStats();
loadMetadataCacheBrowse();
}
}
function closeMetadataCacheModal() {
const modal = document.getElementById('mcache-browse-modal');
if (modal) modal.style.display = 'none';
}
async function loadMetadataCacheBrowseStats() {
try {
const response = await fetch('/api/metadata-cache/stats');
if (!response.ok) return;
const stats = await response.json();
const el = (id, val) => {
const e = document.getElementById(id);
if (e) e.textContent = val;
};
const spotifyTotal = (stats.artists?.spotify || 0) + (stats.albums?.spotify || 0) + (stats.tracks?.spotify || 0);
const itunesTotal = (stats.artists?.itunes || 0) + (stats.albums?.itunes || 0) + (stats.tracks?.itunes || 0);
el('mcache-browse-spotify-count', spotifyTotal);
el('mcache-browse-itunes-count', itunesTotal);
el('mcache-browse-hits', stats.total_hits || 0);
el('mcache-browse-searches', stats.searches || 0);
} catch (e) { /* ignore */ }
}
function switchMetadataCacheTab(tab) {
_mcacheCurrentTab = tab;
_mcachePage = 0;
document.querySelectorAll('.mcache-tab').forEach(t => {
t.classList.toggle('active', t.dataset.tab === tab);
});
loadMetadataCacheBrowse();
}
async function loadMetadataCacheBrowse() {
const grid = document.getElementById('mcache-grid');
if (!grid) return;
const source = document.getElementById('mcache-source-filter')?.value || '';
const search = document.getElementById('mcache-search')?.value || '';
const sort = document.getElementById('mcache-sort-filter')?.value || 'last_accessed_at';
const params = new URLSearchParams({
type: _mcacheCurrentTab,
sort: sort,
sort_dir: sort === 'name' ? 'asc' : 'desc',
offset: _mcachePage * MCACHE_PAGE_SIZE,
limit: MCACHE_PAGE_SIZE
});
if (source) params.set('source', source);
if (search) params.set('search', search);
grid.innerHTML = '<div class="mcache-empty"><div class="mcache-empty-icon">...</div><div class="mcache-empty-sub">Loading...</div></div>';
try {
const response = await fetch(`/api/metadata-cache/browse?${params}`);
if (!response.ok) throw new Error('Failed to load');
const data = await response.json();
if (!data.items || data.items.length === 0) {
grid.innerHTML = `
<div class="mcache-empty">
<div class="mcache-empty-icon">📦</div>
<div class="mcache-empty-title">No cached ${_mcacheCurrentTab}s yet</div>
<div class="mcache-empty-sub">As you search and browse music in SoulSync, API responses will be cached here automatically.</div>
</div>`;
renderMetadataCachePagination(0, 0);
return;
}
renderMetadataCacheGrid(data.items, _mcacheCurrentTab);
renderMetadataCachePagination(data.total, data.offset);
} catch (e) {
grid.innerHTML = '<div class="mcache-empty"><div class="mcache-empty-sub">Failed to load cache data.</div></div>';
}
}
function renderMetadataCacheGrid(items, entityType) {
const grid = document.getElementById('mcache-grid');
if (!grid) return;
grid.innerHTML = items.map(item => {
const source = item.source || 'spotify';
const sourceBadge = `<span class="mcache-source-badge ${source}">${source}</span>`;
const cacheAge = formatCacheAge(item.last_accessed_at);
const hits = item.access_count || 1;
let imageHtml = '';
const isArtist = entityType === 'artist';
const shapeClass = isArtist ? ' artist' : '';
if (item.image_url) {
imageHtml = `<img class="mcache-card-image${shapeClass}" src="${item.image_url}" alt="" loading="lazy" onerror="this.style.display='none';this.nextElementSibling.style.display='flex'"><div class="mcache-card-image-placeholder${shapeClass}" style="display:none">${(item.name || '?')[0].toUpperCase()}</div>`;
} else {
imageHtml = `<div class="mcache-card-image-placeholder${shapeClass}">${(item.name || '?')[0].toUpperCase()}</div>`;
}
let subText = '';
let metaText = '';
if (entityType === 'artist') {
const genres = item.genres ? (typeof item.genres === 'string' ? JSON.parse(item.genres || '[]') : item.genres) : [];
subText = genres.length > 0 ? genres.slice(0, 2).join(', ') : '';
if (item.popularity) metaText = `Pop: ${item.popularity}`;
} else if (entityType === 'album') {
subText = item.artist_name || '';
const parts = [];
if (item.release_date) parts.push(item.release_date.substring(0, 4));
if (item.total_tracks) parts.push(`${item.total_tracks} tracks`);
if (item.album_type) parts.push(item.album_type);
metaText = parts.join(' · ');
} else if (entityType === 'track') {
subText = item.artist_name || '';
const parts = [];
if (item.album_name) parts.push(item.album_name);
if (item.duration_ms) parts.push(formatDuration(item.duration_ms));
metaText = parts.join(' · ');
}
return `
<div class="mcache-card" onclick="openMetadataCacheDetail('${source}', '${entityType}', '${item.entity_id}')">
<div class="mcache-card-top">
${imageHtml}
<div class="mcache-card-info">
<div class="mcache-card-name" title="${(item.name || '').replace(/"/g, '&quot;')}">${item.name || 'Unknown'}</div>
${subText ? `<div class="mcache-card-sub">${subText}</div>` : ''}
${metaText ? `<div class="mcache-card-meta">${metaText}</div>` : ''}
</div>
</div>
<div class="mcache-card-bottom">
${sourceBadge}
<span class="mcache-card-cache-info">${cacheAge} · ${hits}x</span>
</div>
</div>`;
}).join('');
}
function renderMetadataCachePagination(total, offset) {
const container = document.getElementById('mcache-pagination');
if (!container) return;
const totalPages = Math.ceil(total / MCACHE_PAGE_SIZE);
const currentPage = Math.floor(offset / MCACHE_PAGE_SIZE);
if (totalPages <= 1) {
container.innerHTML = total > 0 ? `<span style="font-size:11px;color:rgba(255,255,255,0.3)">${total} result${total !== 1 ? 's' : ''}</span>` : '';
return;
}
let html = '';
html += `<button class="mcache-page-btn" ${currentPage === 0 ? 'disabled' : ''} onclick="_mcachePage=${currentPage - 1};loadMetadataCacheBrowse()"></button>`;
const maxVisible = 7;
let start = Math.max(0, currentPage - Math.floor(maxVisible / 2));
let end = Math.min(totalPages, start + maxVisible);
if (end - start < maxVisible) start = Math.max(0, end - maxVisible);
if (start > 0) {
html += `<button class="mcache-page-btn" onclick="_mcachePage=0;loadMetadataCacheBrowse()">1</button>`;
if (start > 1) html += `<span style="color:rgba(255,255,255,0.2);padding:0 4px">...</span>`;
}
for (let i = start; i < end; i++) {
html += `<button class="mcache-page-btn${i === currentPage ? ' active' : ''}" onclick="_mcachePage=${i};loadMetadataCacheBrowse()">${i + 1}</button>`;
}
if (end < totalPages) {
if (end < totalPages - 1) html += `<span style="color:rgba(255,255,255,0.2);padding:0 4px">...</span>`;
html += `<button class="mcache-page-btn" onclick="_mcachePage=${totalPages - 1};loadMetadataCacheBrowse()">${totalPages}</button>`;
}
html += `<button class="mcache-page-btn" ${currentPage >= totalPages - 1 ? 'disabled' : ''} onclick="_mcachePage=${currentPage + 1};loadMetadataCacheBrowse()"></button>`;
html += `<span style="font-size:11px;color:rgba(255,255,255,0.25);margin-left:8px">${total} total</span>`;
container.innerHTML = html;
}
async function openMetadataCacheDetail(source, entityType, entityId) {
const modal = document.getElementById('mcache-detail-modal');
const body = document.getElementById('mcache-detail-body');
const title = document.getElementById('mcache-detail-title');
if (!modal || !body) return;
modal.style.display = 'flex';
body.innerHTML = '<div style="text-align:center;padding:40px;color:rgba(255,255,255,0.4)">Loading...</div>';
if (title) title.textContent = 'Loading...';
try {
const response = await fetch(`/api/metadata-cache/entity/${source}/${entityType}/${entityId}`);
if (!response.ok) throw new Error('Not found');
const data = await response.json();
if (title) title.textContent = data.name || 'Unknown';
const isArtist = entityType === 'artist';
const shapeClass = isArtist ? ' artist' : '';
let imageHtml = '';
if (data.image_url) {
imageHtml = `<img class="mcache-detail-image${shapeClass}" src="${data.image_url}" alt="" onerror="this.style.display='none';this.nextElementSibling.style.display='flex'"><div class="mcache-detail-image-placeholder${shapeClass}" style="display:none">${(data.name || '?')[0].toUpperCase()}</div>`;
} else {
imageHtml = `<div class="mcache-detail-image-placeholder${shapeClass}">${(data.name || '?')[0].toUpperCase()}</div>`;
}
const sourceBadge = `<span class="mcache-source-badge ${source}">${source}</span>`;
const typeBadge = `<span class="mcache-type-badge">${entityType}</span>`;
// Build structured fields table
let fieldsHtml = '<table class="mcache-detail-table">';
const addRow = (label, value) => {
if (value !== null && value !== undefined && value !== '') {
fieldsHtml += `<tr><td>${label}</td><td>${value}</td></tr>`;
}
};
addRow('Entity ID', data.entity_id);
addRow('Name', data.name);
if (entityType === 'artist') {
const genres = data.genres ? (typeof data.genres === 'string' ? JSON.parse(data.genres || '[]') : data.genres) : [];
if (genres.length) addRow('Genres', genres.join(', '));
if (data.popularity) addRow('Popularity', data.popularity);
if (data.followers) addRow('Followers', data.followers.toLocaleString());
} else if (entityType === 'album') {
addRow('Artist', data.artist_name);
addRow('Release Date', data.release_date);
addRow('Total Tracks', data.total_tracks);
addRow('Album Type', data.album_type);
addRow('Label', data.label);
} else if (entityType === 'track') {
addRow('Artist', data.artist_name);
addRow('Album', data.album_name);
if (data.duration_ms) addRow('Duration', formatDuration(data.duration_ms));
addRow('Track Number', data.track_number);
addRow('Disc Number', data.disc_number);
addRow('Explicit', data.explicit ? 'Yes' : 'No');
addRow('ISRC', data.isrc);
if (data.preview_url) addRow('Preview', `<a href="${data.preview_url}" target="_blank" style="color:var(--accent,#6d5dfc)">Listen</a>`);
}
fieldsHtml += '</table>';
// Cache metadata section
let cacheHtml = '<div class="mcache-detail-section-title">Cache Metadata</div>';
cacheHtml += '<table class="mcache-detail-table">';
if (data.created_at) cacheHtml += `<tr><td>Cached At</td><td>${new Date(data.created_at).toLocaleString()}</td></tr>`;
if (data.last_accessed_at) cacheHtml += `<tr><td>Last Accessed</td><td>${new Date(data.last_accessed_at).toLocaleString()}</td></tr>`;
if (data.access_count) cacheHtml += `<tr><td>Access Count</td><td>${data.access_count}</td></tr>`;
if (data.ttl_days) cacheHtml += `<tr><td>TTL</td><td>${data.ttl_days} days</td></tr>`;
cacheHtml += '</table>';
// Raw JSON section
let rawJsonHtml = '';
if (data.raw_json) {
const rawStr = typeof data.raw_json === 'string' ? data.raw_json : JSON.stringify(data.raw_json, null, 2);
const escapedJson = rawStr.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
rawJsonHtml = `
<div class="mcache-detail-section-title">Raw API Response</div>
<button class="mcache-raw-json-toggle" onclick="this.nextElementSibling.style.display=this.nextElementSibling.style.display==='none'?'block':'none';this.textContent=this.nextElementSibling.style.display==='none'?'Show Raw JSON':'Hide Raw JSON'">Show Raw JSON</button>
<pre class="mcache-raw-json" style="display:none">${escapedJson}</pre>`;
}
body.innerHTML = `
<div class="mcache-detail-hero">
${imageHtml}
<div class="mcache-detail-hero-info">
<div class="mcache-detail-hero-name">${data.name || 'Unknown'}</div>
${entityType !== 'artist' && data.artist_name ? `<div class="mcache-detail-hero-sub">${data.artist_name}</div>` : ''}
<div class="mcache-detail-badges">
${sourceBadge}
${typeBadge}
</div>
</div>
</div>
<div class="mcache-detail-section-title">Details</div>
${fieldsHtml}
${cacheHtml}
${rawJsonHtml}`;
} catch (e) {
body.innerHTML = '<div style="text-align:center;padding:40px;color:rgba(255,255,255,0.4)">Failed to load entity details.</div>';
}
}
function closeMetadataCacheDetail() {
const modal = document.getElementById('mcache-detail-modal');
if (modal) modal.style.display = 'none';
}
async function clearMetadataCache() {
if (!confirm('Clear all cached metadata? This will remove all cached API responses from Spotify and iTunes.')) return;
try {
const response = await fetch('/api/metadata-cache/clear', { method: 'DELETE' });
const data = await response.json();
if (data.success) {
showToast(`Cleared ${data.cleared} cached entries`, 'success');
loadMetadataCacheBrowseStats();
loadMetadataCacheBrowse();
loadMetadataCacheStats();
} else {
showToast('Failed to clear cache', 'error');
}
} catch (e) {
showToast('Error clearing cache', 'error');
}
}
function debouncedMetadataCacheSearch() {
if (_mcacheSearchTimeout) clearTimeout(_mcacheSearchTimeout);
_mcacheSearchTimeout = setTimeout(() => {
_mcachePage = 0;
loadMetadataCacheBrowse();
}, 400);
}
function formatCacheAge(timestamp) {
if (!timestamp) return '—';
const now = new Date();
const then = new Date(timestamp);
const diffMs = now - then;
const diffMin = Math.floor(diffMs / 60000);
if (diffMin < 1) return 'now';
if (diffMin < 60) return `${diffMin}m`;
const diffHr = Math.floor(diffMin / 60);
if (diffHr < 24) return `${diffHr}h`;
const diffDays = Math.floor(diffHr / 24);
if (diffDays < 30) return `${diffDays}d`;
return `${Math.floor(diffDays / 30)}mo`;
}
// ============================================
// == TOOL HELP MODAL ==
// ============================================
@ -18704,6 +19082,34 @@ const TOOL_HELP_CONTENT = {
<li><strong>DB Size:</strong> Current size of the live database</li>
</ul>
`
},
'metadata-cache': {
title: 'Metadata Cache',
content: `
<h4>What is this?</h4>
<p>The Metadata Cache stores every API response from Spotify and iTunes so SoulSync can reuse them instead of making duplicate API calls. This reduces rate limit pressure and speeds up lookups.</p>
<h4>How it works</h4>
<p>When SoulSync fetches artist, album, or track data from Spotify or iTunes, the response is cached locally. The next time the same data is needed, it's served from cache instantly no API call required. Cached data is even served during Spotify rate limit bans.</p>
<h4>Browsing the Cache</h4>
<p>Click <strong>Browse Cache</strong> to explore all cached metadata. You can filter by entity type (artists, albums, tracks), search by name, filter by source (Spotify/iTunes), and sort by different fields. Click any card to see full details including the raw API response.</p>
<h4>Cache Management</h4>
<ul>
<li><strong>TTL:</strong> Entities expire after 30 days, search mappings after 7 days</li>
<li><strong>Eviction:</strong> Expired entries are automatically cleaned up</li>
<li><strong>Clear:</strong> You can clear the entire cache or filter by source/type</li>
</ul>
<h4>Stats Explained</h4>
<ul>
<li><strong>Artists:</strong> Total cached artist profiles</li>
<li><strong>Albums:</strong> Total cached album records</li>
<li><strong>Tracks:</strong> Total cached track records</li>
<li><strong>Hits:</strong> Total number of times cached data was served instead of making an API call</li>
</ul>
`
}
};
@ -19426,6 +19832,9 @@ async function loadDashboardData() {
// Initial load of discovery pool stats for the tool card
loadDiscoveryPoolStats();
// Initial load of metadata cache stats
loadMetadataCacheStats();
// Initial load of wishlist count
await updateWishlistCount();

View file

@ -38249,4 +38249,655 @@ tr.tag-diff-same {
.issue-detail-snapshot-value {
max-width: 200px;
}
}
/* ============================================ */
/* == METADATA CACHE BROWSER == */
/* ============================================ */
.mcache-modal-overlay {
position: fixed;
top: 0; left: 0; right: 0; bottom: 0;
background: rgba(0, 0, 0, 0.85);
z-index: 10000;
display: flex;
align-items: center;
justify-content: center;
animation: mcache-fadeIn 0.2s ease;
}
@keyframes mcache-fadeIn {
from { opacity: 0; }
to { opacity: 1; }
}
.mcache-modal {
width: 95vw;
max-width: 1400px;
height: 95vh;
max-height: 95vh;
background: rgba(14, 14, 14, 0.98);
border: 1px solid rgba(255, 255, 255, 0.08);
border-radius: 16px;
display: flex;
flex-direction: column;
overflow: hidden;
box-shadow: 0 25px 80px rgba(0, 0, 0, 0.6);
}
.mcache-modal-header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 18px 24px;
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
flex-shrink: 0;
}
.mcache-modal-title {
margin: 0;
font-size: 17px;
font-weight: 600;
color: #fff;
}
.mcache-modal-header-actions {
display: flex;
align-items: center;
gap: 10px;
}
.mcache-btn-clear {
background: rgba(239, 68, 68, 0.12);
color: #ef4444;
border: 1px solid rgba(239, 68, 68, 0.2);
border-radius: 8px;
padding: 6px 14px;
font-size: 12px;
font-weight: 500;
cursor: pointer;
transition: all 0.2s ease;
}
.mcache-btn-clear:hover {
background: rgba(239, 68, 68, 0.2);
border-color: rgba(239, 68, 68, 0.35);
}
.mcache-modal-close {
background: none;
border: none;
color: rgba(255, 255, 255, 0.4);
font-size: 24px;
cursor: pointer;
padding: 0 4px;
line-height: 1;
transition: color 0.2s;
}
.mcache-modal-close:hover {
color: #fff;
}
/* Stats Bar */
.mcache-stats-bar {
display: flex;
gap: 10px;
padding: 14px 24px;
border-bottom: 1px solid rgba(255, 255, 255, 0.06);
flex-shrink: 0;
flex-wrap: wrap;
}
.mcache-stat-pill {
background: rgba(255, 255, 255, 0.04);
border: 1px solid rgba(255, 255, 255, 0.06);
border-radius: 10px;
padding: 8px 16px;
display: flex;
flex-direction: column;
align-items: center;
gap: 2px;
min-width: 100px;
}
.mcache-stat-pill-label {
font-size: 10px;
text-transform: uppercase;
letter-spacing: 0.5px;
color: rgba(255, 255, 255, 0.4);
}
.mcache-stat-pill-value {
font-size: 18px;
font-weight: 600;
color: var(--accent, #6d5dfc);
}
/* Tabs */
.mcache-tabs {
display: flex;
gap: 0;
padding: 0 24px;
border-bottom: 1px solid rgba(255, 255, 255, 0.06);
flex-shrink: 0;
}
.mcache-tab {
background: none;
border: none;
border-bottom: 2px solid transparent;
color: rgba(255, 255, 255, 0.45);
font-size: 13px;
font-weight: 500;
padding: 12px 20px;
cursor: pointer;
transition: all 0.2s ease;
}
.mcache-tab:hover {
color: rgba(255, 255, 255, 0.7);
}
.mcache-tab.active {
color: var(--accent, #6d5dfc);
border-bottom-color: var(--accent, #6d5dfc);
}
/* Filters */
.mcache-filters {
display: flex;
gap: 10px;
padding: 14px 24px;
flex-shrink: 0;
flex-wrap: wrap;
}
.mcache-search-input {
flex: 1;
min-width: 200px;
background: rgba(255, 255, 255, 0.05);
border: 1px solid rgba(255, 255, 255, 0.08);
border-radius: 8px;
padding: 9px 14px;
color: #fff;
font-size: 13px;
outline: none;
transition: border-color 0.2s;
}
.mcache-search-input:focus {
border-color: rgba(var(--accent-rgb, 109, 93, 252), 0.4);
}
.mcache-search-input::placeholder {
color: rgba(255, 255, 255, 0.25);
}
.mcache-source-filter,
.mcache-sort-filter {
background: rgba(255, 255, 255, 0.05);
border: 1px solid rgba(255, 255, 255, 0.08);
border-radius: 8px;
padding: 9px 12px;
color: #fff;
font-size: 12px;
cursor: pointer;
outline: none;
}
.mcache-source-filter option,
.mcache-sort-filter option {
background: #1a1a1a;
color: #fff;
}
/* Grid */
.mcache-grid {
flex: 1;
overflow-y: auto;
padding: 16px 24px;
display: grid;
grid-template-columns: repeat(auto-fill, minmax(220px, 1fr));
gap: 12px;
align-content: start;
}
.mcache-grid::-webkit-scrollbar {
width: 6px;
}
.mcache-grid::-webkit-scrollbar-thumb {
background: rgba(255, 255, 255, 0.12);
border-radius: 3px;
}
/* Cards */
.mcache-card {
background: rgba(22, 22, 22, 0.95);
border: 1px solid rgba(255, 255, 255, 0.06);
border-radius: 10px;
padding: 14px;
cursor: pointer;
transition: all 0.2s ease;
display: flex;
flex-direction: column;
gap: 8px;
}
.mcache-card:hover {
border-color: rgba(var(--accent-rgb, 109, 93, 252), 0.25);
transform: translateY(-2px);
box-shadow: 0 6px 20px rgba(0, 0, 0, 0.3);
}
.mcache-card-top {
display: flex;
gap: 12px;
align-items: flex-start;
}
.mcache-card-image {
width: 56px;
height: 56px;
border-radius: 6px;
object-fit: cover;
flex-shrink: 0;
background: rgba(255, 255, 255, 0.05);
}
.mcache-card-image.artist {
border-radius: 50%;
}
.mcache-card-image-placeholder {
width: 56px;
height: 56px;
border-radius: 6px;
flex-shrink: 0;
display: flex;
align-items: center;
justify-content: center;
font-size: 18px;
font-weight: 700;
color: rgba(255, 255, 255, 0.5);
background: linear-gradient(135deg, rgba(var(--accent-rgb, 109, 93, 252), 0.2), rgba(var(--accent-rgb, 109, 93, 252), 0.05));
}
.mcache-card-image-placeholder.artist {
border-radius: 50%;
}
.mcache-card-info {
flex: 1;
min-width: 0;
display: flex;
flex-direction: column;
gap: 2px;
}
.mcache-card-name {
font-size: 13px;
font-weight: 600;
color: #fff;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.mcache-card-sub {
font-size: 11px;
color: rgba(255, 255, 255, 0.4);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.mcache-card-meta {
font-size: 11px;
color: rgba(255, 255, 255, 0.3);
}
.mcache-card-bottom {
display: flex;
align-items: center;
justify-content: space-between;
gap: 6px;
padding-top: 6px;
border-top: 1px solid rgba(255, 255, 255, 0.04);
}
.mcache-source-badge {
font-size: 9px;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.3px;
padding: 2px 7px;
border-radius: 8px;
}
.mcache-source-badge.spotify {
background: rgba(30, 215, 96, 0.15);
color: #1ed760;
}
.mcache-source-badge.itunes {
background: rgba(252, 60, 68, 0.15);
color: #fc3c44;
}
.mcache-card-cache-info {
font-size: 10px;
color: rgba(255, 255, 255, 0.25);
}
/* Type badges */
.mcache-type-badge {
font-size: 9px;
font-weight: 500;
text-transform: uppercase;
letter-spacing: 0.3px;
padding: 1px 6px;
border-radius: 6px;
background: rgba(255, 255, 255, 0.06);
color: rgba(255, 255, 255, 0.4);
}
/* Empty state */
.mcache-empty {
grid-column: 1 / -1;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
padding: 60px 20px;
text-align: center;
}
.mcache-empty-icon {
font-size: 48px;
margin-bottom: 16px;
opacity: 0.3;
}
.mcache-empty-title {
font-size: 18px;
font-weight: 600;
background: linear-gradient(135deg, var(--accent, #6d5dfc), rgba(var(--accent-rgb, 109, 93, 252), 0.5));
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
background-clip: text;
margin-bottom: 8px;
}
.mcache-empty-sub {
font-size: 13px;
color: rgba(255, 255, 255, 0.35);
max-width: 340px;
}
/* Pagination */
.mcache-pagination {
display: flex;
align-items: center;
justify-content: center;
gap: 6px;
padding: 14px 24px;
border-top: 1px solid rgba(255, 255, 255, 0.06);
flex-shrink: 0;
}
.mcache-page-btn {
background: rgba(255, 255, 255, 0.05);
border: 1px solid rgba(255, 255, 255, 0.08);
border-radius: 6px;
color: rgba(255, 255, 255, 0.5);
font-size: 12px;
padding: 6px 10px;
cursor: pointer;
transition: all 0.2s;
min-width: 32px;
text-align: center;
}
.mcache-page-btn:hover {
background: rgba(255, 255, 255, 0.08);
color: #fff;
}
.mcache-page-btn.active {
background: rgba(var(--accent-rgb, 109, 93, 252), 0.2);
border-color: rgba(var(--accent-rgb, 109, 93, 252), 0.3);
color: var(--accent, #6d5dfc);
}
.mcache-page-btn:disabled {
opacity: 0.3;
cursor: not-allowed;
}
/* Detail Modal */
.mcache-detail-overlay {
position: fixed;
top: 0; left: 0; right: 0; bottom: 0;
background: rgba(0, 0, 0, 0.6);
z-index: 10001;
display: flex;
align-items: center;
justify-content: center;
animation: mcache-fadeIn 0.15s ease;
}
.mcache-detail-modal {
width: 90vw;
max-width: 700px;
max-height: 85vh;
background: rgba(18, 18, 18, 0.99);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 14px;
display: flex;
flex-direction: column;
overflow: hidden;
box-shadow: 0 20px 60px rgba(0, 0, 0, 0.5);
}
.mcache-detail-header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 16px 20px;
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
flex-shrink: 0;
}
.mcache-detail-title {
margin: 0;
font-size: 15px;
font-weight: 600;
color: #fff;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.mcache-detail-body {
flex: 1;
overflow-y: auto;
padding: 20px;
}
.mcache-detail-body::-webkit-scrollbar {
width: 6px;
}
.mcache-detail-body::-webkit-scrollbar-thumb {
background: rgba(255, 255, 255, 0.12);
border-radius: 3px;
}
.mcache-detail-hero {
display: flex;
gap: 20px;
align-items: flex-start;
margin-bottom: 20px;
}
.mcache-detail-image {
width: 160px;
height: 160px;
border-radius: 10px;
object-fit: cover;
flex-shrink: 0;
background: rgba(255, 255, 255, 0.05);
}
.mcache-detail-image.artist {
border-radius: 50%;
}
.mcache-detail-image-placeholder {
width: 160px;
height: 160px;
border-radius: 10px;
flex-shrink: 0;
display: flex;
align-items: center;
justify-content: center;
font-size: 48px;
font-weight: 700;
color: rgba(255, 255, 255, 0.4);
background: linear-gradient(135deg, rgba(var(--accent-rgb, 109, 93, 252), 0.25), rgba(var(--accent-rgb, 109, 93, 252), 0.05));
}
.mcache-detail-image-placeholder.artist {
border-radius: 50%;
}
.mcache-detail-hero-info {
flex: 1;
min-width: 0;
}
.mcache-detail-hero-name {
font-size: 22px;
font-weight: 700;
color: #fff;
margin-bottom: 6px;
word-wrap: break-word;
}
.mcache-detail-hero-sub {
font-size: 13px;
color: rgba(255, 255, 255, 0.5);
margin-bottom: 10px;
}
.mcache-detail-badges {
display: flex;
gap: 6px;
flex-wrap: wrap;
margin-bottom: 10px;
}
.mcache-detail-table {
width: 100%;
border-collapse: collapse;
margin-bottom: 20px;
}
.mcache-detail-table tr {
border-bottom: 1px solid rgba(255, 255, 255, 0.04);
}
.mcache-detail-table td {
padding: 8px 0;
font-size: 12px;
vertical-align: top;
}
.mcache-detail-table td:first-child {
color: rgba(255, 255, 255, 0.4);
width: 130px;
font-weight: 500;
padding-right: 12px;
}
.mcache-detail-table td:last-child {
color: rgba(255, 255, 255, 0.8);
word-break: break-word;
}
.mcache-detail-section-title {
font-size: 12px;
font-weight: 600;
color: rgba(255, 255, 255, 0.5);
text-transform: uppercase;
letter-spacing: 0.5px;
margin: 20px 0 10px;
padding-bottom: 6px;
border-bottom: 1px solid rgba(255, 255, 255, 0.06);
}
.mcache-raw-json-toggle {
background: rgba(255, 255, 255, 0.04);
border: 1px solid rgba(255, 255, 255, 0.08);
border-radius: 8px;
color: rgba(255, 255, 255, 0.5);
font-size: 12px;
padding: 8px 14px;
cursor: pointer;
width: 100%;
text-align: left;
transition: all 0.2s;
margin-bottom: 8px;
}
.mcache-raw-json-toggle:hover {
background: rgba(255, 255, 255, 0.06);
color: rgba(255, 255, 255, 0.7);
}
.mcache-raw-json {
background: rgba(0, 0, 0, 0.4);
border: 1px solid rgba(255, 255, 255, 0.06);
border-radius: 8px;
padding: 14px;
font-family: 'SF Mono', 'Fira Code', 'Cascadia Code', monospace;
font-size: 11px;
color: rgba(255, 255, 255, 0.65);
overflow-x: auto;
white-space: pre-wrap;
word-break: break-all;
max-height: 400px;
overflow-y: auto;
line-height: 1.5;
}
/* Responsive */
@media (max-width: 768px) {
.mcache-modal {
width: 100vw;
height: 100vh;
border-radius: 0;
}
.mcache-grid {
grid-template-columns: repeat(auto-fill, minmax(160px, 1fr));
}
.mcache-detail-hero {
flex-direction: column;
align-items: center;
text-align: center;
}
.mcache-detail-image,
.mcache-detail-image-placeholder {
width: 120px;
height: 120px;
}
.mcache-filters {
flex-direction: column;
}
}