danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Harden metadata cache: prevent simplified data from overwriting full entries, fix connection leaks, and add inline TTL enforcement

Browse source
This commit is contained in:
Broque Thomas 2026-03-14 13:39:12 -07:00
parent 0b8bfa1e6b
commit cf917279c2
3 changed files with 411 additions and 347 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
core/itunes_client.py
View file

@ -403,31 +403,48 @@ class iTunesClient:
""" """
Perform a batched lookup of artist IDs to get clean artist names. Perform a batched lookup of artist IDs to get clean artist names.
Returns a map of {artist_id: clean_artist_name} Returns a map of {artist_id: clean_artist_name}
Checks cache first to avoid unnecessary API calls.
""" """
if not artist_ids: if not artist_ids:
return {} return {}
clean_names = {} clean_names = {}
uncached_ids = []
# Check cache first
cache = get_metadata_cache()
for aid in artist_ids:
cached = cache.get_entity('itunes', 'artist', aid)
if cached and cached.get('artistName'):
clean_names[aid] = cached['artistName']
else:
uncached_ids.append(aid)
if not uncached_ids:
return clean_names
# iTunes lookup allows comma-separated IDs, but keep batch size reasonable (e.g. 50) # iTunes lookup allows comma-separated IDs, but keep batch size reasonable (e.g. 50)
batch_size = 50 batch_size = 50
for i in range(0, len(artist_ids), batch_size): for i in range(0, len(uncached_ids), batch_size):
batch = artist_ids[i:i+batch_size] batch = uncached_ids[i:i+batch_size]
ids_str = ",".join(batch) ids_str = ",".join(batch)
try: try:
# Lookup is fast/unlimited compared to search # Lookup is fast/unlimited compared to search
results = self._lookup(id=ids_str) results = self._lookup(id=ids_str)
for item in results: for item in results:
if item.get('wrapperType') == 'artist': if item.get('wrapperType') == 'artist':
a_id = str(item.get('artistId', '')) a_id = str(item.get('artistId', ''))
a_name = item.get('artistName', '') a_name = item.get('artistName', '')
if a_id and a_name: if a_id and a_name:
clean_names[a_id] = a_name clean_names[a_id] = a_name
# Populate artist cache from lookup results
cache.store_entity('itunes', 'artist', a_id, item)
except Exception as e: except Exception as e:
logger.warning(f"Failed batch artist lookup: {e}") logger.warning(f"Failed batch artist lookup: {e}")
return clean_names return clean_names
def get_track_details(self, track_id: str) -> Optional[Dict[str, Any]]: def get_track_details(self, track_id: str) -> Optional[Dict[str, Any]]:
@ -571,10 +588,10 @@ class iTunesClient:
result = albums[:limit] result = albums[:limit]
# Cache individual albums + search mapping # Cache individual albums + search mapping (skip if full data already cached)
entries = [(str(ad.get('collectionId', '')), ad) for ad in raw_items if ad.get('collectionId')] entries = [(str(ad.get('collectionId', '')), ad) for ad in raw_items if ad.get('collectionId')]
if entries: if entries:
cache.store_entities_bulk('itunes', 'album', entries) cache.store_entities_bulk('itunes', 'album', entries, skip_if_exists=True)
# Only cache IDs for the albums we're actually returning # Only cache IDs for the albums we're actually returning
result_ids = [str(ad.get('collectionId', '')) for ad in raw_items[:limit] if ad.get('collectionId')] result_ids = [str(ad.get('collectionId', '')) for ad in raw_items[:limit] if ad.get('collectionId')]
if result_ids: if result_ids:
@ -782,13 +799,13 @@ class iTunesClient:
# Cache the album tracks listing # Cache the album tracks listing
cache.store_entity('itunes', 'album', f"{album_id}_tracks", result) cache.store_entity('itunes', 'album', f"{album_id}_tracks", result)
# Also cache individual tracks from the raw results # Also cache individual tracks from the raw results (skip if full data already cached)
track_entries = [] track_entries = []
for item in results: for item in results:
if item.get('wrapperType') == 'track' and item.get('kind') == 'song' and item.get('trackId'): if item.get('wrapperType') == 'track' and item.get('kind') == 'song' and item.get('trackId'):
track_entries.append((str(item['trackId']), item)) track_entries.append((str(item['trackId']), item))
if track_entries: if track_entries:
cache.store_entities_bulk('itunes', 'track', track_entries) cache.store_entities_bulk('itunes', 'track', track_entries, skip_if_exists=True)
return result return result
@ -1032,14 +1049,14 @@ class iTunesClient:
# Extract albums from dict # Extract albums from dict
albums = [item['album'] for item in seen_albums.values()] albums = [item['album'] for item in seen_albums.values()]
# Cache individual albums opportunistically # Cache individual albums opportunistically (skip if full data already cached)
album_entries = [] album_entries = []
for album_data in results: for album_data in results:
if album_data.get('wrapperType') == 'collection' and album_data.get('collectionId'): if album_data.get('wrapperType') == 'collection' and album_data.get('collectionId'):
album_entries.append((str(album_data['collectionId']), album_data)) album_entries.append((str(album_data['collectionId']), album_data))
if album_entries: if album_entries:
cache = get_metadata_cache() cache = get_metadata_cache()
cache.store_entities_bulk('itunes', 'album', album_entries) cache.store_entities_bulk('itunes', 'album', album_entries, skip_if_exists=True)
logger.info(f"Retrieved {len(albums)} unique albums for artist {artist_id} (filtered from {len(results)} results)") logger.info(f"Retrieved {len(albums)} unique albums for artist {artist_id} (filtered from {len(results)} results)")
return albums[:limit] return albums[:limit]

703
core/metadata_cache.py
View file

@ -46,24 +46,36 @@ class MetadataCache:
try: try:
db = self._get_db() db = self._get_db()
conn = db._get_connection() conn = db._get_connection()
cursor = conn.cursor() try:
cursor.execute(""" cursor = conn.cursor()
SELECT id, raw_json FROM metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id = ?
""", (source, entity_type, entity_id))
row = cursor.fetchone()
if row:
# Touch: update access stats
cursor.execute(""" cursor.execute("""
UPDATE metadata_cache_entities SELECT id, raw_json, updated_at, ttl_days FROM metadata_cache_entities
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1 WHERE source = ? AND entity_type = ? AND entity_id = ?
WHERE id = ? """, (source, entity_type, entity_id))
""", (row['id'],)) row = cursor.fetchone()
conn.commit() if row:
# Inline TTL check — don't serve stale data
try:
updated = datetime.fromisoformat(row['updated_at'])
age_days = (datetime.now() - updated).days
if age_days > (row['ttl_days'] or 30):
cursor.execute("DELETE FROM metadata_cache_entities WHERE id = ?", (row['id'],))
conn.commit()
return None
except (ValueError, TypeError):
pass
# Touch: update access stats
cursor.execute("""
UPDATE metadata_cache_entities
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
WHERE id = ?
""", (row['id'],))
conn.commit()
return json.loads(row['raw_json'])
return None
finally:
conn.close() conn.close()
return json.loads(row['raw_json'])
conn.close()
return None
except Exception as e: except Exception as e:
logger.debug(f"Cache lookup error ({source}/{entity_type}/{entity_id}): {e}") logger.debug(f"Cache lookup error ({source}/{entity_type}/{entity_id}): {e}")
return None return None
@ -77,76 +89,22 @@ class MetadataCache:
raw_json = json.dumps(raw_data, default=str) raw_json = json.dumps(raw_data, default=str)
db = self._get_db() db = self._get_db()
conn = db._get_connection() conn = db._get_connection()
cursor = conn.cursor() try:
cursor.execute(""" cursor = conn.cursor()
INSERT OR REPLACE INTO metadata_cache_entities
(source, entity_type, entity_id, name, image_url, external_urls,
genres, popularity, followers,
artist_name, artist_id, release_date, total_tracks, album_type, label,
album_name, album_id, duration_ms, track_number, disc_number, explicit, isrc, preview_url,
raw_json, updated_at, last_accessed_at, access_count)
VALUES (?, ?, ?, ?, ?, ?,
?, ?, ?,
?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?,
?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP,
COALESCE((SELECT access_count FROM metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id = ?), 0) + 1)
""", (
source, entity_type, entity_id,
fields.get('name', ''),
fields.get('image_url'),
fields.get('external_urls'),
fields.get('genres'),
fields.get('popularity'),
fields.get('followers'),
fields.get('artist_name'),
fields.get('artist_id'),
fields.get('release_date'),
fields.get('total_tracks'),
fields.get('album_type'),
fields.get('label'),
fields.get('album_name'),
fields.get('album_id'),
fields.get('duration_ms'),
fields.get('track_number'),
fields.get('disc_number'),
fields.get('explicit'),
fields.get('isrc'),
fields.get('preview_url'),
raw_json,
source, entity_type, entity_id,
))
conn.commit()
conn.close()
except Exception as e:
logger.debug(f"Cache store error ({source}/{entity_type}/{entity_id}): {e}")
def store_entities_bulk(self, source: str, entity_type: str, items: List[Tuple[str, dict]]) -> None:
"""Store multiple entities at once. items = [(entity_id, raw_data), ...]"""
if not items:
return
try:
db = self._get_db()
conn = db._get_connection()
cursor = conn.cursor()
for entity_id, raw_data in items:
if not entity_id or not raw_data:
continue
fields = self._extract_fields(source, entity_type, raw_data)
raw_json = json.dumps(raw_data, default=str)
cursor.execute(""" cursor.execute("""
INSERT OR REPLACE INTO metadata_cache_entities INSERT OR REPLACE INTO metadata_cache_entities
(source, entity_type, entity_id, name, image_url, external_urls, (source, entity_type, entity_id, name, image_url, external_urls,
genres, popularity, followers, genres, popularity, followers,
artist_name, artist_id, release_date, total_tracks, album_type, label, artist_name, artist_id, release_date, total_tracks, album_type, label,
album_name, album_id, duration_ms, track_number, disc_number, explicit, isrc, preview_url, album_name, album_id, duration_ms, track_number, disc_number, explicit, isrc, preview_url,
raw_json, updated_at, last_accessed_at) raw_json, updated_at, last_accessed_at, access_count)
VALUES (?, ?, ?, ?, ?, ?, VALUES (?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP,
COALESCE((SELECT access_count FROM metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id = ?), 0) + 1)
""", ( """, (
source, entity_type, entity_id, source, entity_type, entity_id,
fields.get('name', ''), fields.get('name', ''),
@ -170,9 +128,86 @@ class MetadataCache:
fields.get('isrc'), fields.get('isrc'),
fields.get('preview_url'), fields.get('preview_url'),
raw_json, raw_json,
source, entity_type, entity_id,
)) ))
conn.commit() conn.commit()
conn.close() finally:
conn.close()
except Exception as e:
logger.debug(f"Cache store error ({source}/{entity_type}/{entity_id}): {e}")
def store_entities_bulk(self, source: str, entity_type: str, items: List[Tuple[str, dict]],
skip_if_exists: bool = False) -> None:
"""Store multiple entities at once. items = [(entity_id, raw_data), ...]
Args:
skip_if_exists: If True, don't overwrite existing entries. Use this for
opportunistic caching of simplified data (e.g. from list endpoints)
to avoid replacing richer data from detail endpoints.
"""
if not items:
return
try:
db = self._get_db()
conn = db._get_connection()
try:
cursor = conn.cursor()
for entity_id, raw_data in items:
if not entity_id or not raw_data:
continue
if skip_if_exists:
cursor.execute("""
SELECT 1 FROM metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id = ?
""", (source, entity_type, entity_id))
if cursor.fetchone():
continue
fields = self._extract_fields(source, entity_type, raw_data)
raw_json = json.dumps(raw_data, default=str)
cursor.execute("""
INSERT OR REPLACE INTO metadata_cache_entities
(source, entity_type, entity_id, name, image_url, external_urls,
genres, popularity, followers,
artist_name, artist_id, release_date, total_tracks, album_type, label,
album_name, album_id, duration_ms, track_number, disc_number, explicit, isrc, preview_url,
raw_json, updated_at, last_accessed_at, access_count)
VALUES (?, ?, ?, ?, ?, ?,
?, ?, ?,
?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?,
?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP,
COALESCE((SELECT access_count FROM metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id = ?), 0) + 1)
""", (
source, entity_type, entity_id,
fields.get('name', ''),
fields.get('image_url'),
fields.get('external_urls'),
fields.get('genres'),
fields.get('popularity'),
fields.get('followers'),
fields.get('artist_name'),
fields.get('artist_id'),
fields.get('release_date'),
fields.get('total_tracks'),
fields.get('album_type'),
fields.get('label'),
fields.get('album_name'),
fields.get('album_id'),
fields.get('duration_ms'),
fields.get('track_number'),
fields.get('disc_number'),
fields.get('explicit'),
fields.get('isrc'),
fields.get('preview_url'),
raw_json,
source, entity_type, entity_id,
))
conn.commit()
finally:
conn.close()
except Exception as e: except Exception as e:
logger.debug(f"Cache bulk store error ({source}/{entity_type}): {e}") logger.debug(f"Cache bulk store error ({source}/{entity_type}): {e}")
@ -186,29 +221,31 @@ class MetadataCache:
try: try:
db = self._get_db() db = self._get_db()
conn = db._get_connection() conn = db._get_connection()
cursor = conn.cursor() try:
# Batch query in chunks of 500 to avoid SQLite variable limit cursor = conn.cursor()
for i in range(0, len(entity_ids), 500): # Batch query in chunks of 500 to avoid SQLite variable limit
chunk = entity_ids[i:i + 500] for i in range(0, len(entity_ids), 500):
placeholders = ','.join('?' * len(chunk)) chunk = entity_ids[i:i + 500]
cursor.execute(f""" placeholders = ','.join('?' * len(chunk))
SELECT entity_id, raw_json FROM metadata_cache_entities cursor.execute(f"""
WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders}) SELECT entity_id, raw_json FROM metadata_cache_entities
""", [source, entity_type] + chunk) WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders})
for row in cursor.fetchall(): """, [source, entity_type] + chunk)
found[row['entity_id']] = json.loads(row['raw_json']) for row in cursor.fetchall():
# Touch all found entries found[row['entity_id']] = json.loads(row['raw_json'])
if found: # Touch all found entries
found_in_chunk = [eid for eid in chunk if eid in found] if found:
if found_in_chunk: found_in_chunk = [eid for eid in chunk if eid in found]
ph2 = ','.join('?' * len(found_in_chunk)) if found_in_chunk:
cursor.execute(f""" ph2 = ','.join('?' * len(found_in_chunk))
UPDATE metadata_cache_entities cursor.execute(f"""
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1 UPDATE metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id IN ({ph2}) SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
""", [source, entity_type] + found_in_chunk) WHERE source = ? AND entity_type = ? AND entity_id IN ({ph2})
conn.commit() """, [source, entity_type] + found_in_chunk)
conn.close() conn.commit()
finally:
conn.close()
missing = [eid for eid in entity_ids if eid not in found] missing = [eid for eid in entity_ids if eid not in found]
except Exception as e: except Exception as e:
logger.debug(f"Cache batch lookup error: {e}") logger.debug(f"Cache batch lookup error: {e}")
@ -226,58 +263,57 @@ class MetadataCache:
try: try:
db = self._get_db() db = self._get_db()
conn = db._get_connection() conn = db._get_connection()
cursor = conn.cursor()
cursor.execute("""
SELECT id, result_ids, created_at FROM metadata_cache_searches
WHERE source = ? AND search_type = ? AND query_normalized = ? AND search_limit = ?
""", (source, search_type, normalized, limit))
row = cursor.fetchone()
if not row:
conn.close()
return None
# Check TTL (7 days for searches)
try: try:
created = datetime.fromisoformat(row['created_at']) cursor = conn.cursor()
age_days = (datetime.now() - created).days
if age_days > 7:
# Expired — delete and return miss
cursor.execute("DELETE FROM metadata_cache_searches WHERE id = ?", (row['id'],))
conn.commit()
conn.close()
return None
except (ValueError, TypeError):
pass
# Touch search entry
cursor.execute("""
UPDATE metadata_cache_searches
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
WHERE id = ?
""", (row['id'],))
conn.commit()
# Resolve entity IDs to full data
result_ids = json.loads(row['result_ids'])
if not result_ids:
conn.close()
return []
results = []
for eid in result_ids:
cursor.execute(""" cursor.execute("""
SELECT raw_json FROM metadata_cache_entities SELECT id, result_ids, created_at FROM metadata_cache_searches
WHERE source = ? AND entity_type = ? AND entity_id = ? WHERE source = ? AND search_type = ? AND query_normalized = ? AND search_limit = ?
""", (source, search_type, eid)) """, (source, search_type, normalized, limit))
erow = cursor.fetchone() row = cursor.fetchone()
if erow: if not row:
results.append(json.loads(erow['raw_json'])) return None
conn.close()
# Only return if we found all (or most) entries — partial results are unreliable # Check TTL (7 days for searches)
if len(results) >= len(result_ids) * 0.8: try:
return results created = datetime.fromisoformat(row['created_at'])
return None age_days = (datetime.now() - created).days
if age_days > 7:
# Expired — delete and return miss
cursor.execute("DELETE FROM metadata_cache_searches WHERE id = ?", (row['id'],))
conn.commit()
return None
except (ValueError, TypeError):
pass
# Touch search entry
cursor.execute("""
UPDATE metadata_cache_searches
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
WHERE id = ?
""", (row['id'],))
conn.commit()
# Resolve entity IDs to full data
result_ids = json.loads(row['result_ids'])
if not result_ids:
return []
results = []
for eid in result_ids:
cursor.execute("""
SELECT raw_json FROM metadata_cache_entities
WHERE source = ? AND entity_type = ? AND entity_id = ?
""", (source, search_type, eid))
erow = cursor.fetchone()
if erow:
results.append(json.loads(erow['raw_json']))
# Only return if we found all (or most) entries — partial results are unreliable
if len(results) >= len(result_ids) * 0.8:
return results
return None
finally:
conn.close()
except Exception as e: except Exception as e:
logger.debug(f"Search cache lookup error ({source}/{search_type}/{query}): {e}") logger.debug(f"Search cache lookup error ({source}/{search_type}/{query}): {e}")
return None return None
@ -291,18 +327,20 @@ class MetadataCache:
try: try:
db = self._get_db() db = self._get_db()
conn = db._get_connection() conn = db._get_connection()
cursor = conn.cursor() try:
cursor.execute(""" cursor = conn.cursor()
INSERT OR REPLACE INTO metadata_cache_searches cursor.execute("""
(source, search_type, query_normalized, query_original, result_ids, INSERT OR REPLACE INTO metadata_cache_searches
result_count, search_limit, last_accessed_at) (source, search_type, query_normalized, query_original, result_ids,
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) result_count, search_limit, last_accessed_at)
""", ( VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
source, search_type, normalized, query.strip(), """, (
json.dumps(entity_ids), len(entity_ids), limit, source, search_type, normalized, query.strip(),
)) json.dumps(entity_ids), len(entity_ids), limit,
conn.commit() ))
conn.close() conn.commit()
finally:
conn.close()
except Exception as e: except Exception as e:
logger.debug(f"Search cache store error ({source}/{search_type}/{query}): {e}") logger.debug(f"Search cache store error ({source}/{search_type}/{query}): {e}")
@ -315,63 +353,65 @@ class MetadataCache:
try: try:
db = self._get_db() db = self._get_db()
conn = db._get_connection() conn = db._get_connection()
cursor = conn.cursor() try:
cursor = conn.cursor()
where_clauses = ['entity_type = ?'] where_clauses = ['entity_type = ?']
params = [entity_type] params = [entity_type]
# Exclude pseudo-entities like album_id_tracks and track_id_features # Exclude pseudo-entities like album_id_tracks and track_id_features
where_clauses.append(r"entity_id NOT LIKE '%\_tracks' ESCAPE '\'") where_clauses.append(r"entity_id NOT LIKE '%\_tracks' ESCAPE '\'")
where_clauses.append(r"entity_id NOT LIKE '%\_features' ESCAPE '\'") where_clauses.append(r"entity_id NOT LIKE '%\_features' ESCAPE '\'")
if source: if source:
where_clauses.append('source = ?') where_clauses.append('source = ?')
params.append(source) params.append(source)
if search: if search:
search_term = f'%{search}%' search_term = f'%{search}%'
where_clauses.append('(name LIKE ? OR artist_name LIKE ? OR album_name LIKE ?)') where_clauses.append('(name LIKE ? OR artist_name LIKE ? OR album_name LIKE ?)')
params.extend([search_term, search_term, search_term]) params.extend([search_term, search_term, search_term])
where_sql = ' AND '.join(where_clauses) where_sql = ' AND '.join(where_clauses)
# Count total # Count total
cursor.execute(f"SELECT COUNT(*) as cnt FROM metadata_cache_entities WHERE {where_sql}", params) cursor.execute(f"SELECT COUNT(*) as cnt FROM metadata_cache_entities WHERE {where_sql}", params)
total = cursor.fetchone()['cnt'] total = cursor.fetchone()['cnt']
# Validate sort column # Validate sort column
valid_sorts = {'last_accessed_at', 'created_at', 'access_count', 'name', 'popularity', 'updated_at'} valid_sorts = {'last_accessed_at', 'created_at', 'access_count', 'name', 'popularity', 'updated_at'}
if sort not in valid_sorts: if sort not in valid_sorts:
sort = 'last_accessed_at' sort = 'last_accessed_at'
direction = 'ASC' if sort_dir == 'asc' else 'DESC' direction = 'ASC' if sort_dir == 'asc' else 'DESC'
cursor.execute(f""" cursor.execute(f"""
SELECT id, source, entity_type, entity_id, name, image_url, SELECT id, source, entity_type, entity_id, name, image_url,
genres, popularity, followers, genres, popularity, followers,
artist_name, artist_id, release_date, total_tracks, album_type, label, artist_name, artist_id, release_date, total_tracks, album_type, label,
album_name, album_id, duration_ms, track_number, disc_number, explicit, album_name, album_id, duration_ms, track_number, disc_number, explicit,
isrc, preview_url, external_urls, isrc, preview_url, external_urls,
created_at, updated_at, last_accessed_at, access_count created_at, updated_at, last_accessed_at, access_count
FROM metadata_cache_entities FROM metadata_cache_entities
WHERE {where_sql} WHERE {where_sql}
ORDER BY {sort} {direction} ORDER BY {sort} {direction}
LIMIT ? OFFSET ? LIMIT ? OFFSET ?
""", params + [limit, offset]) """, params + [limit, offset])
items = [] items = []
for row in cursor.fetchall(): for row in cursor.fetchall():
item = dict(row) item = dict(row)
# Parse JSON fields for the UI # Parse JSON fields for the UI
for json_field in ('genres', 'external_urls'): for json_field in ('genres', 'external_urls'):
if item.get(json_field): if item.get(json_field):
try: try:
item[json_field] = json.loads(item[json_field]) item[json_field] = json.loads(item[json_field])
except (json.JSONDecodeError, TypeError): except (json.JSONDecodeError, TypeError):
pass pass
items.append(item) items.append(item)
conn.close() return {'items': items, 'total': total, 'offset': offset, 'limit': limit}
return {'items': items, 'total': total, 'offset': offset, 'limit': limit} finally:
conn.close()
except Exception as e: except Exception as e:
logger.error(f"Cache browse error: {e}") logger.error(f"Cache browse error: {e}")
return {'items': [], 'total': 0, 'offset': offset, 'limit': limit} return {'items': [], 'total': 0, 'offset': offset, 'limit': limit}
@ -381,34 +421,35 @@ class MetadataCache:
try: try:
db = self._get_db() db = self._get_db()
conn = db._get_connection() conn = db._get_connection()
cursor = conn.cursor() try:
cursor.execute(""" cursor = conn.cursor()
SELECT * FROM metadata_cache_entities cursor.execute("""
WHERE source = ? AND entity_type = ? AND entity_id = ? SELECT * FROM metadata_cache_entities
""", (source, entity_type, entity_id)) WHERE source = ? AND entity_type = ? AND entity_id = ?
row = cursor.fetchone() """, (source, entity_type, entity_id))
if not row: row = cursor.fetchone()
if not row:
return None
# Touch
cursor.execute("""
UPDATE metadata_cache_entities
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
WHERE id = ?
""", (row['id'],))
conn.commit()
item = dict(row)
# Parse JSON fields
for json_field in ('genres', 'external_urls', 'raw_json'):
if item.get(json_field):
try:
item[json_field] = json.loads(item[json_field])
except (json.JSONDecodeError, TypeError):
pass
return item
finally:
conn.close() conn.close()
return None
# Touch
cursor.execute("""
UPDATE metadata_cache_entities
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
WHERE id = ?
""", (row['id'],))
conn.commit()
item = dict(row)
# Parse JSON fields
for json_field in ('genres', 'external_urls', 'raw_json'):
if item.get(json_field):
try:
item[json_field] = json.loads(item[json_field])
except (json.JSONDecodeError, TypeError):
pass
conn.close()
return item
except Exception as e: except Exception as e:
logger.error(f"Cache detail error: {e}") logger.error(f"Cache detail error: {e}")
return None return None
@ -420,49 +461,51 @@ class MetadataCache:
try: try:
db = self._get_db() db = self._get_db()
conn = db._get_connection() conn = db._get_connection()
cursor = conn.cursor() try:
cursor = conn.cursor()
stats = { stats = {
'artists': {'spotify': 0, 'itunes': 0}, 'artists': {'spotify': 0, 'itunes': 0},
'albums': {'spotify': 0, 'itunes': 0}, 'albums': {'spotify': 0, 'itunes': 0},
'tracks': {'spotify': 0, 'itunes': 0}, 'tracks': {'spotify': 0, 'itunes': 0},
'searches': 0, 'searches': 0,
'total_entries': 0, 'total_entries': 0,
'total_hits': 0, 'total_hits': 0,
'oldest': None, 'oldest': None,
'newest': None, 'newest': None,
} }
# Count by type and source (exclude pseudo-entities like _tracks, _features) # Count by type and source (exclude pseudo-entities like _tracks, _features)
cursor.execute(r""" cursor.execute(r"""
SELECT entity_type, source, COUNT(*) as cnt, SUM(access_count) as hits SELECT entity_type, source, COUNT(*) as cnt, SUM(access_count) as hits
FROM metadata_cache_entities FROM metadata_cache_entities
WHERE entity_id NOT LIKE '%\_tracks' ESCAPE '\' WHERE entity_id NOT LIKE '%\_tracks' ESCAPE '\'
AND entity_id NOT LIKE '%\_features' ESCAPE '\' AND entity_id NOT LIKE '%\_features' ESCAPE '\'
GROUP BY entity_type, source GROUP BY entity_type, source
""") """)
type_key_map = {'artist': 'artists', 'album': 'albums', 'track': 'tracks'} type_key_map = {'artist': 'artists', 'album': 'albums', 'track': 'tracks'}
for row in cursor.fetchall(): for row in cursor.fetchall():
et = type_key_map.get(row['entity_type']) et = type_key_map.get(row['entity_type'])
src = row['source'] src = row['source']
if et and et in stats and src in stats[et]: if et and et in stats and src in stats[et]:
stats[et][src] = row['cnt'] stats[et][src] = row['cnt']
stats['total_entries'] += row['cnt'] stats['total_entries'] += row['cnt']
stats['total_hits'] += (row['hits'] or 0) stats['total_hits'] += (row['hits'] or 0)
# Search count # Search count
cursor.execute("SELECT COUNT(*) as cnt FROM metadata_cache_searches") cursor.execute("SELECT COUNT(*) as cnt FROM metadata_cache_searches")
stats['searches'] = cursor.fetchone()['cnt'] stats['searches'] = cursor.fetchone()['cnt']
# Oldest and newest # Oldest and newest
cursor.execute("SELECT MIN(created_at) as oldest, MAX(created_at) as newest FROM metadata_cache_entities") cursor.execute("SELECT MIN(created_at) as oldest, MAX(created_at) as newest FROM metadata_cache_entities")
row = cursor.fetchone() row = cursor.fetchone()
if row: if row:
stats['oldest'] = row['oldest'] stats['oldest'] = row['oldest']
stats['newest'] = row['newest'] stats['newest'] = row['newest']
conn.close() return stats
return stats finally:
conn.close()
except Exception as e: except Exception as e:
logger.error(f"Cache stats error: {e}") logger.error(f"Cache stats error: {e}")
return { return {
@ -480,28 +523,30 @@ class MetadataCache:
try: try:
db = self._get_db() db = self._get_db()
conn = db._get_connection() conn = db._get_connection()
cursor = conn.cursor() try:
cursor = conn.cursor()
# Entities # Entities
cursor.execute(""" cursor.execute("""
DELETE FROM metadata_cache_entities DELETE FROM metadata_cache_entities
WHERE julianday('now') - julianday(updated_at) > ttl_days WHERE julianday('now') - julianday(updated_at) > ttl_days
""") """)
entity_count = cursor.rowcount entity_count = cursor.rowcount
# Searches # Searches
cursor.execute(""" cursor.execute("""
DELETE FROM metadata_cache_searches DELETE FROM metadata_cache_searches
WHERE julianday('now') - julianday(created_at) > ttl_days WHERE julianday('now') - julianday(created_at) > ttl_days
""") """)
search_count = cursor.rowcount search_count = cursor.rowcount
conn.commit() conn.commit()
conn.close() total = entity_count + search_count
total = entity_count + search_count if total > 0:
if total > 0: logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)")
logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)") return total
return total finally:
conn.close()
except Exception as e: except Exception as e:
logger.error(f"Cache eviction error: {e}") logger.error(f"Cache eviction error: {e}")
return 0 return 0
@ -511,46 +556,48 @@ class MetadataCache:
try: try:
db = self._get_db() db = self._get_db()
conn = db._get_connection() conn = db._get_connection()
cursor = conn.cursor() try:
cursor = conn.cursor()
# Clear entities # Clear entities
where_parts = [] where_parts = []
params = [] params = []
if source: if source:
where_parts.append('source = ?') where_parts.append('source = ?')
params.append(source) params.append(source)
if entity_type: if entity_type:
where_parts.append('entity_type = ?') where_parts.append('entity_type = ?')
params.append(entity_type) params.append(entity_type)
if where_parts: if where_parts:
where_sql = ' AND '.join(where_parts) where_sql = ' AND '.join(where_parts)
cursor.execute(f"DELETE FROM metadata_cache_entities WHERE {where_sql}", params) cursor.execute(f"DELETE FROM metadata_cache_entities WHERE {where_sql}", params)
else: else:
cursor.execute("DELETE FROM metadata_cache_entities") cursor.execute("DELETE FROM metadata_cache_entities")
entity_count = cursor.rowcount entity_count = cursor.rowcount
# Clear searches (match source and entity_type → search_type) # Clear searches (match source and entity_type → search_type)
search_where = [] search_where = []
search_params = [] search_params = []
if source: if source:
search_where.append('source = ?') search_where.append('source = ?')
search_params.append(source) search_params.append(source)
if entity_type: if entity_type:
search_where.append('search_type = ?') search_where.append('search_type = ?')
search_params.append(entity_type) search_params.append(entity_type)
if search_where: if search_where:
cursor.execute(f"DELETE FROM metadata_cache_searches WHERE {' AND '.join(search_where)}", search_params) cursor.execute(f"DELETE FROM metadata_cache_searches WHERE {' AND '.join(search_where)}", search_params)
else: else:
cursor.execute("DELETE FROM metadata_cache_searches") cursor.execute("DELETE FROM metadata_cache_searches")
search_count = cursor.rowcount search_count = cursor.rowcount
conn.commit() conn.commit()
conn.close() total = entity_count + search_count
total = entity_count + search_count logger.info(f"Cleared {total} cache entries (source={source}, type={entity_type})")
logger.info(f"Cleared {total} cache entries (source={source}, type={entity_type})") return total
return total finally:
conn.close()
except Exception as e: except Exception as e:
logger.error(f"Cache clear error: {e}") logger.error(f"Cache clear error: {e}")
return 0 return 0

12
core/spotify_client.py
View file

@ -1049,10 +1049,10 @@ class SpotifyClient:
album = Album.from_spotify_album(album_data) album = Album.from_spotify_album(album_data)
albums.append(album) albums.append(album)
# Cache individual albums + search mapping # Cache individual albums + search mapping (skip if full data already cached)
entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')] entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')]
if entries: if entries:
cache.store_entities_bulk('spotify', 'album', entries) cache.store_entities_bulk('spotify', 'album', entries, skip_if_exists=True)
cache.store_search_results('spotify', 'album', query, min(limit, 10), cache.store_search_results('spotify', 'album', query, min(limit, 10),
[ad.get('id') for ad in raw_items if ad.get('id')]) [ad.get('id') for ad in raw_items if ad.get('id')])
@ -1234,14 +1234,14 @@ class SpotifyClient:
# Cache the aggregated result # Cache the aggregated result
cache.store_entity('spotify', 'album', cache_key, result) cache.store_entity('spotify', 'album', cache_key, result)
# Also cache individual tracks opportunistically # Also cache individual tracks opportunistically (skip if full data already cached)
track_entries = [] track_entries = []
for track in all_tracks: for track in all_tracks:
tid = track.get('id') tid = track.get('id')
if tid: if tid:
track_entries.append((tid, track)) track_entries.append((tid, track))
if track_entries: if track_entries:
cache.store_entities_bulk('spotify', 'track', track_entries) cache.store_entities_bulk('spotify', 'track', track_entries, skip_if_exists=True)
return result return result
@ -1278,11 +1278,11 @@ class SpotifyClient:
logger.info(f"Retrieved {len(albums)} albums for artist {artist_id}") logger.info(f"Retrieved {len(albums)} albums for artist {artist_id}")
# Cache individual albums opportunistically # Cache individual albums opportunistically (skip if full data already cached)
cache = get_metadata_cache() cache = get_metadata_cache()
entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')] entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')]
if entries: if entries:
cache.store_entities_bulk('spotify', 'album', entries) cache.store_entities_bulk('spotify', 'album', entries, skip_if_exists=True)
return albums return albums