Harden metadata cache: prevent simplified data from overwriting full entries, fix connection leaks, and add inline TTL enforcement
This commit is contained in:
Broque Thomas
parent
0b8bfa1e6b
commit
cf917279c2
3 changed files with 411 additions and 347 deletions
|
|
@ -403,31 +403,48 @@ class iTunesClient:
|
|||
"""
|
||||
Perform a batched lookup of artist IDs to get clean artist names.
|
||||
Returns a map of {artist_id: clean_artist_name}
|
||||
Checks cache first to avoid unnecessary API calls.
|
||||
"""
|
||||
if not artist_ids:
|
||||
return {}
|
||||
|
||||
|
||||
clean_names = {}
|
||||
uncached_ids = []
|
||||
|
||||
# Check cache first
|
||||
cache = get_metadata_cache()
|
||||
for aid in artist_ids:
|
||||
cached = cache.get_entity('itunes', 'artist', aid)
|
||||
if cached and cached.get('artistName'):
|
||||
clean_names[aid] = cached['artistName']
|
||||
else:
|
||||
uncached_ids.append(aid)
|
||||
|
||||
if not uncached_ids:
|
||||
return clean_names
|
||||
|
||||
# iTunes lookup allows comma-separated IDs, but keep batch size reasonable (e.g. 50)
|
||||
batch_size = 50
|
||||
|
||||
for i in range(0, len(artist_ids), batch_size):
|
||||
batch = artist_ids[i:i+batch_size]
|
||||
|
||||
for i in range(0, len(uncached_ids), batch_size):
|
||||
batch = uncached_ids[i:i+batch_size]
|
||||
ids_str = ",".join(batch)
|
||||
|
||||
|
||||
try:
|
||||
# Lookup is fast/unlimited compared to search
|
||||
results = self._lookup(id=ids_str)
|
||||
|
||||
|
||||
for item in results:
|
||||
if item.get('wrapperType') == 'artist':
|
||||
a_id = str(item.get('artistId', ''))
|
||||
a_name = item.get('artistName', '')
|
||||
if a_id and a_name:
|
||||
clean_names[a_id] = a_name
|
||||
# Populate artist cache from lookup results
|
||||
cache.store_entity('itunes', 'artist', a_id, item)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed batch artist lookup: {e}")
|
||||
|
||||
|
||||
return clean_names
|
||||
|
||||
def get_track_details(self, track_id: str) -> Optional[Dict[str, Any]]:
|
||||
|
|
@ -571,10 +588,10 @@ class iTunesClient:
|
|||
|
||||
result = albums[:limit]
|
||||
|
||||
# Cache individual albums + search mapping
|
||||
# Cache individual albums + search mapping (skip if full data already cached)
|
||||
entries = [(str(ad.get('collectionId', '')), ad) for ad in raw_items if ad.get('collectionId')]
|
||||
if entries:
|
||||
cache.store_entities_bulk('itunes', 'album', entries)
|
||||
cache.store_entities_bulk('itunes', 'album', entries, skip_if_exists=True)
|
||||
# Only cache IDs for the albums we're actually returning
|
||||
result_ids = [str(ad.get('collectionId', '')) for ad in raw_items[:limit] if ad.get('collectionId')]
|
||||
if result_ids:
|
||||
|
|
@ -782,13 +799,13 @@ class iTunesClient:
|
|||
# Cache the album tracks listing
|
||||
cache.store_entity('itunes', 'album', f"{album_id}_tracks", result)
|
||||
|
||||
# Also cache individual tracks from the raw results
|
||||
# Also cache individual tracks from the raw results (skip if full data already cached)
|
||||
track_entries = []
|
||||
for item in results:
|
||||
if item.get('wrapperType') == 'track' and item.get('kind') == 'song' and item.get('trackId'):
|
||||
track_entries.append((str(item['trackId']), item))
|
||||
if track_entries:
|
||||
cache.store_entities_bulk('itunes', 'track', track_entries)
|
||||
cache.store_entities_bulk('itunes', 'track', track_entries, skip_if_exists=True)
|
||||
|
||||
return result
|
||||
|
||||
|
|
@ -1032,14 +1049,14 @@ class iTunesClient:
|
|||
# Extract albums from dict
|
||||
albums = [item['album'] for item in seen_albums.values()]
|
||||
|
||||
# Cache individual albums opportunistically
|
||||
# Cache individual albums opportunistically (skip if full data already cached)
|
||||
album_entries = []
|
||||
for album_data in results:
|
||||
if album_data.get('wrapperType') == 'collection' and album_data.get('collectionId'):
|
||||
album_entries.append((str(album_data['collectionId']), album_data))
|
||||
if album_entries:
|
||||
cache = get_metadata_cache()
|
||||
cache.store_entities_bulk('itunes', 'album', album_entries)
|
||||
cache.store_entities_bulk('itunes', 'album', album_entries, skip_if_exists=True)
|
||||
|
||||
logger.info(f"Retrieved {len(albums)} unique albums for artist {artist_id} (filtered from {len(results)} results)")
|
||||
return albums[:limit]
|
||||
|
|
|
|||
|
|
@ -46,24 +46,36 @@ class MetadataCache:
|
|||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT id, raw_json FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id = ?
|
||||
""", (source, entity_type, entity_id))
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
# Touch: update access stats
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE metadata_cache_entities
|
||||
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
|
||||
WHERE id = ?
|
||||
""", (row['id'],))
|
||||
conn.commit()
|
||||
SELECT id, raw_json, updated_at, ttl_days FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id = ?
|
||||
""", (source, entity_type, entity_id))
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
# Inline TTL check — don't serve stale data
|
||||
try:
|
||||
updated = datetime.fromisoformat(row['updated_at'])
|
||||
age_days = (datetime.now() - updated).days
|
||||
if age_days > (row['ttl_days'] or 30):
|
||||
cursor.execute("DELETE FROM metadata_cache_entities WHERE id = ?", (row['id'],))
|
||||
conn.commit()
|
||||
return None
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Touch: update access stats
|
||||
cursor.execute("""
|
||||
UPDATE metadata_cache_entities
|
||||
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
|
||||
WHERE id = ?
|
||||
""", (row['id'],))
|
||||
conn.commit()
|
||||
return json.loads(row['raw_json'])
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
return json.loads(row['raw_json'])
|
||||
conn.close()
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"Cache lookup error ({source}/{entity_type}/{entity_id}): {e}")
|
||||
return None
|
||||
|
|
@ -77,76 +89,22 @@ class MetadataCache:
|
|||
raw_json = json.dumps(raw_data, default=str)
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO metadata_cache_entities
|
||||
(source, entity_type, entity_id, name, image_url, external_urls,
|
||||
genres, popularity, followers,
|
||||
artist_name, artist_id, release_date, total_tracks, album_type, label,
|
||||
album_name, album_id, duration_ms, track_number, disc_number, explicit, isrc, preview_url,
|
||||
raw_json, updated_at, last_accessed_at, access_count)
|
||||
VALUES (?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP,
|
||||
COALESCE((SELECT access_count FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id = ?), 0) + 1)
|
||||
""", (
|
||||
source, entity_type, entity_id,
|
||||
fields.get('name', ''),
|
||||
fields.get('image_url'),
|
||||
fields.get('external_urls'),
|
||||
fields.get('genres'),
|
||||
fields.get('popularity'),
|
||||
fields.get('followers'),
|
||||
fields.get('artist_name'),
|
||||
fields.get('artist_id'),
|
||||
fields.get('release_date'),
|
||||
fields.get('total_tracks'),
|
||||
fields.get('album_type'),
|
||||
fields.get('label'),
|
||||
fields.get('album_name'),
|
||||
fields.get('album_id'),
|
||||
fields.get('duration_ms'),
|
||||
fields.get('track_number'),
|
||||
fields.get('disc_number'),
|
||||
fields.get('explicit'),
|
||||
fields.get('isrc'),
|
||||
fields.get('preview_url'),
|
||||
raw_json,
|
||||
source, entity_type, entity_id,
|
||||
))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.debug(f"Cache store error ({source}/{entity_type}/{entity_id}): {e}")
|
||||
|
||||
def store_entities_bulk(self, source: str, entity_type: str, items: List[Tuple[str, dict]]) -> None:
|
||||
"""Store multiple entities at once. items = [(entity_id, raw_data), ...]"""
|
||||
if not items:
|
||||
return
|
||||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
for entity_id, raw_data in items:
|
||||
if not entity_id or not raw_data:
|
||||
continue
|
||||
fields = self._extract_fields(source, entity_type, raw_data)
|
||||
raw_json = json.dumps(raw_data, default=str)
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO metadata_cache_entities
|
||||
(source, entity_type, entity_id, name, image_url, external_urls,
|
||||
genres, popularity, followers,
|
||||
artist_name, artist_id, release_date, total_tracks, album_type, label,
|
||||
album_name, album_id, duration_ms, track_number, disc_number, explicit, isrc, preview_url,
|
||||
raw_json, updated_at, last_accessed_at)
|
||||
raw_json, updated_at, last_accessed_at, access_count)
|
||||
VALUES (?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP,
|
||||
COALESCE((SELECT access_count FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id = ?), 0) + 1)
|
||||
""", (
|
||||
source, entity_type, entity_id,
|
||||
fields.get('name', ''),
|
||||
|
|
@ -170,9 +128,86 @@ class MetadataCache:
|
|||
fields.get('isrc'),
|
||||
fields.get('preview_url'),
|
||||
raw_json,
|
||||
source, entity_type, entity_id,
|
||||
))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.debug(f"Cache store error ({source}/{entity_type}/{entity_id}): {e}")
|
||||
|
||||
def store_entities_bulk(self, source: str, entity_type: str, items: List[Tuple[str, dict]],
|
||||
skip_if_exists: bool = False) -> None:
|
||||
"""Store multiple entities at once. items = [(entity_id, raw_data), ...]
|
||||
|
||||
Args:
|
||||
skip_if_exists: If True, don't overwrite existing entries. Use this for
|
||||
opportunistic caching of simplified data (e.g. from list endpoints)
|
||||
to avoid replacing richer data from detail endpoints.
|
||||
"""
|
||||
if not items:
|
||||
return
|
||||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
for entity_id, raw_data in items:
|
||||
if not entity_id or not raw_data:
|
||||
continue
|
||||
|
||||
if skip_if_exists:
|
||||
cursor.execute("""
|
||||
SELECT 1 FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id = ?
|
||||
""", (source, entity_type, entity_id))
|
||||
if cursor.fetchone():
|
||||
continue
|
||||
|
||||
fields = self._extract_fields(source, entity_type, raw_data)
|
||||
raw_json = json.dumps(raw_data, default=str)
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO metadata_cache_entities
|
||||
(source, entity_type, entity_id, name, image_url, external_urls,
|
||||
genres, popularity, followers,
|
||||
artist_name, artist_id, release_date, total_tracks, album_type, label,
|
||||
album_name, album_id, duration_ms, track_number, disc_number, explicit, isrc, preview_url,
|
||||
raw_json, updated_at, last_accessed_at, access_count)
|
||||
VALUES (?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP,
|
||||
COALESCE((SELECT access_count FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id = ?), 0) + 1)
|
||||
""", (
|
||||
source, entity_type, entity_id,
|
||||
fields.get('name', ''),
|
||||
fields.get('image_url'),
|
||||
fields.get('external_urls'),
|
||||
fields.get('genres'),
|
||||
fields.get('popularity'),
|
||||
fields.get('followers'),
|
||||
fields.get('artist_name'),
|
||||
fields.get('artist_id'),
|
||||
fields.get('release_date'),
|
||||
fields.get('total_tracks'),
|
||||
fields.get('album_type'),
|
||||
fields.get('label'),
|
||||
fields.get('album_name'),
|
||||
fields.get('album_id'),
|
||||
fields.get('duration_ms'),
|
||||
fields.get('track_number'),
|
||||
fields.get('disc_number'),
|
||||
fields.get('explicit'),
|
||||
fields.get('isrc'),
|
||||
fields.get('preview_url'),
|
||||
raw_json,
|
||||
source, entity_type, entity_id,
|
||||
))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.debug(f"Cache bulk store error ({source}/{entity_type}): {e}")
|
||||
|
||||
|
|
@ -186,29 +221,31 @@ class MetadataCache:
|
|||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
# Batch query in chunks of 500 to avoid SQLite variable limit
|
||||
for i in range(0, len(entity_ids), 500):
|
||||
chunk = entity_ids[i:i + 500]
|
||||
placeholders = ','.join('?' * len(chunk))
|
||||
cursor.execute(f"""
|
||||
SELECT entity_id, raw_json FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders})
|
||||
""", [source, entity_type] + chunk)
|
||||
for row in cursor.fetchall():
|
||||
found[row['entity_id']] = json.loads(row['raw_json'])
|
||||
# Touch all found entries
|
||||
if found:
|
||||
found_in_chunk = [eid for eid in chunk if eid in found]
|
||||
if found_in_chunk:
|
||||
ph2 = ','.join('?' * len(found_in_chunk))
|
||||
cursor.execute(f"""
|
||||
UPDATE metadata_cache_entities
|
||||
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
|
||||
WHERE source = ? AND entity_type = ? AND entity_id IN ({ph2})
|
||||
""", [source, entity_type] + found_in_chunk)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
# Batch query in chunks of 500 to avoid SQLite variable limit
|
||||
for i in range(0, len(entity_ids), 500):
|
||||
chunk = entity_ids[i:i + 500]
|
||||
placeholders = ','.join('?' * len(chunk))
|
||||
cursor.execute(f"""
|
||||
SELECT entity_id, raw_json FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders})
|
||||
""", [source, entity_type] + chunk)
|
||||
for row in cursor.fetchall():
|
||||
found[row['entity_id']] = json.loads(row['raw_json'])
|
||||
# Touch all found entries
|
||||
if found:
|
||||
found_in_chunk = [eid for eid in chunk if eid in found]
|
||||
if found_in_chunk:
|
||||
ph2 = ','.join('?' * len(found_in_chunk))
|
||||
cursor.execute(f"""
|
||||
UPDATE metadata_cache_entities
|
||||
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
|
||||
WHERE source = ? AND entity_type = ? AND entity_id IN ({ph2})
|
||||
""", [source, entity_type] + found_in_chunk)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
missing = [eid for eid in entity_ids if eid not in found]
|
||||
except Exception as e:
|
||||
logger.debug(f"Cache batch lookup error: {e}")
|
||||
|
|
@ -226,58 +263,57 @@ class MetadataCache:
|
|||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT id, result_ids, created_at FROM metadata_cache_searches
|
||||
WHERE source = ? AND search_type = ? AND query_normalized = ? AND search_limit = ?
|
||||
""", (source, search_type, normalized, limit))
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
conn.close()
|
||||
return None
|
||||
|
||||
# Check TTL (7 days for searches)
|
||||
try:
|
||||
created = datetime.fromisoformat(row['created_at'])
|
||||
age_days = (datetime.now() - created).days
|
||||
if age_days > 7:
|
||||
# Expired — delete and return miss
|
||||
cursor.execute("DELETE FROM metadata_cache_searches WHERE id = ?", (row['id'],))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return None
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Touch search entry
|
||||
cursor.execute("""
|
||||
UPDATE metadata_cache_searches
|
||||
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
|
||||
WHERE id = ?
|
||||
""", (row['id'],))
|
||||
conn.commit()
|
||||
|
||||
# Resolve entity IDs to full data
|
||||
result_ids = json.loads(row['result_ids'])
|
||||
if not result_ids:
|
||||
conn.close()
|
||||
return []
|
||||
|
||||
results = []
|
||||
for eid in result_ids:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT raw_json FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id = ?
|
||||
""", (source, search_type, eid))
|
||||
erow = cursor.fetchone()
|
||||
if erow:
|
||||
results.append(json.loads(erow['raw_json']))
|
||||
conn.close()
|
||||
SELECT id, result_ids, created_at FROM metadata_cache_searches
|
||||
WHERE source = ? AND search_type = ? AND query_normalized = ? AND search_limit = ?
|
||||
""", (source, search_type, normalized, limit))
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
|
||||
# Only return if we found all (or most) entries — partial results are unreliable
|
||||
if len(results) >= len(result_ids) * 0.8:
|
||||
return results
|
||||
return None
|
||||
# Check TTL (7 days for searches)
|
||||
try:
|
||||
created = datetime.fromisoformat(row['created_at'])
|
||||
age_days = (datetime.now() - created).days
|
||||
if age_days > 7:
|
||||
# Expired — delete and return miss
|
||||
cursor.execute("DELETE FROM metadata_cache_searches WHERE id = ?", (row['id'],))
|
||||
conn.commit()
|
||||
return None
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Touch search entry
|
||||
cursor.execute("""
|
||||
UPDATE metadata_cache_searches
|
||||
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
|
||||
WHERE id = ?
|
||||
""", (row['id'],))
|
||||
conn.commit()
|
||||
|
||||
# Resolve entity IDs to full data
|
||||
result_ids = json.loads(row['result_ids'])
|
||||
if not result_ids:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for eid in result_ids:
|
||||
cursor.execute("""
|
||||
SELECT raw_json FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id = ?
|
||||
""", (source, search_type, eid))
|
||||
erow = cursor.fetchone()
|
||||
if erow:
|
||||
results.append(json.loads(erow['raw_json']))
|
||||
|
||||
# Only return if we found all (or most) entries — partial results are unreliable
|
||||
if len(results) >= len(result_ids) * 0.8:
|
||||
return results
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.debug(f"Search cache lookup error ({source}/{search_type}/{query}): {e}")
|
||||
return None
|
||||
|
|
@ -291,18 +327,20 @@ class MetadataCache:
|
|||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO metadata_cache_searches
|
||||
(source, search_type, query_normalized, query_original, result_ids,
|
||||
result_count, search_limit, last_accessed_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
""", (
|
||||
source, search_type, normalized, query.strip(),
|
||||
json.dumps(entity_ids), len(entity_ids), limit,
|
||||
))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO metadata_cache_searches
|
||||
(source, search_type, query_normalized, query_original, result_ids,
|
||||
result_count, search_limit, last_accessed_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
""", (
|
||||
source, search_type, normalized, query.strip(),
|
||||
json.dumps(entity_ids), len(entity_ids), limit,
|
||||
))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.debug(f"Search cache store error ({source}/{search_type}/{query}): {e}")
|
||||
|
||||
|
|
@ -315,63 +353,65 @@ class MetadataCache:
|
|||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
|
||||
where_clauses = ['entity_type = ?']
|
||||
params = [entity_type]
|
||||
where_clauses = ['entity_type = ?']
|
||||
params = [entity_type]
|
||||
|
||||
# Exclude pseudo-entities like album_id_tracks and track_id_features
|
||||
where_clauses.append(r"entity_id NOT LIKE '%\_tracks' ESCAPE '\'")
|
||||
where_clauses.append(r"entity_id NOT LIKE '%\_features' ESCAPE '\'")
|
||||
# Exclude pseudo-entities like album_id_tracks and track_id_features
|
||||
where_clauses.append(r"entity_id NOT LIKE '%\_tracks' ESCAPE '\'")
|
||||
where_clauses.append(r"entity_id NOT LIKE '%\_features' ESCAPE '\'")
|
||||
|
||||
if source:
|
||||
where_clauses.append('source = ?')
|
||||
params.append(source)
|
||||
if source:
|
||||
where_clauses.append('source = ?')
|
||||
params.append(source)
|
||||
|
||||
if search:
|
||||
search_term = f'%{search}%'
|
||||
where_clauses.append('(name LIKE ? OR artist_name LIKE ? OR album_name LIKE ?)')
|
||||
params.extend([search_term, search_term, search_term])
|
||||
if search:
|
||||
search_term = f'%{search}%'
|
||||
where_clauses.append('(name LIKE ? OR artist_name LIKE ? OR album_name LIKE ?)')
|
||||
params.extend([search_term, search_term, search_term])
|
||||
|
||||
where_sql = ' AND '.join(where_clauses)
|
||||
where_sql = ' AND '.join(where_clauses)
|
||||
|
||||
# Count total
|
||||
cursor.execute(f"SELECT COUNT(*) as cnt FROM metadata_cache_entities WHERE {where_sql}", params)
|
||||
total = cursor.fetchone()['cnt']
|
||||
# Count total
|
||||
cursor.execute(f"SELECT COUNT(*) as cnt FROM metadata_cache_entities WHERE {where_sql}", params)
|
||||
total = cursor.fetchone()['cnt']
|
||||
|
||||
# Validate sort column
|
||||
valid_sorts = {'last_accessed_at', 'created_at', 'access_count', 'name', 'popularity', 'updated_at'}
|
||||
if sort not in valid_sorts:
|
||||
sort = 'last_accessed_at'
|
||||
direction = 'ASC' if sort_dir == 'asc' else 'DESC'
|
||||
# Validate sort column
|
||||
valid_sorts = {'last_accessed_at', 'created_at', 'access_count', 'name', 'popularity', 'updated_at'}
|
||||
if sort not in valid_sorts:
|
||||
sort = 'last_accessed_at'
|
||||
direction = 'ASC' if sort_dir == 'asc' else 'DESC'
|
||||
|
||||
cursor.execute(f"""
|
||||
SELECT id, source, entity_type, entity_id, name, image_url,
|
||||
genres, popularity, followers,
|
||||
artist_name, artist_id, release_date, total_tracks, album_type, label,
|
||||
album_name, album_id, duration_ms, track_number, disc_number, explicit,
|
||||
isrc, preview_url, external_urls,
|
||||
created_at, updated_at, last_accessed_at, access_count
|
||||
FROM metadata_cache_entities
|
||||
WHERE {where_sql}
|
||||
ORDER BY {sort} {direction}
|
||||
LIMIT ? OFFSET ?
|
||||
""", params + [limit, offset])
|
||||
cursor.execute(f"""
|
||||
SELECT id, source, entity_type, entity_id, name, image_url,
|
||||
genres, popularity, followers,
|
||||
artist_name, artist_id, release_date, total_tracks, album_type, label,
|
||||
album_name, album_id, duration_ms, track_number, disc_number, explicit,
|
||||
isrc, preview_url, external_urls,
|
||||
created_at, updated_at, last_accessed_at, access_count
|
||||
FROM metadata_cache_entities
|
||||
WHERE {where_sql}
|
||||
ORDER BY {sort} {direction}
|
||||
LIMIT ? OFFSET ?
|
||||
""", params + [limit, offset])
|
||||
|
||||
items = []
|
||||
for row in cursor.fetchall():
|
||||
item = dict(row)
|
||||
# Parse JSON fields for the UI
|
||||
for json_field in ('genres', 'external_urls'):
|
||||
if item.get(json_field):
|
||||
try:
|
||||
item[json_field] = json.loads(item[json_field])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
items.append(item)
|
||||
items = []
|
||||
for row in cursor.fetchall():
|
||||
item = dict(row)
|
||||
# Parse JSON fields for the UI
|
||||
for json_field in ('genres', 'external_urls'):
|
||||
if item.get(json_field):
|
||||
try:
|
||||
item[json_field] = json.loads(item[json_field])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
items.append(item)
|
||||
|
||||
conn.close()
|
||||
return {'items': items, 'total': total, 'offset': offset, 'limit': limit}
|
||||
return {'items': items, 'total': total, 'offset': offset, 'limit': limit}
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Cache browse error: {e}")
|
||||
return {'items': [], 'total': 0, 'offset': offset, 'limit': limit}
|
||||
|
|
@ -381,34 +421,35 @@ class MetadataCache:
|
|||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT * FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id = ?
|
||||
""", (source, entity_type, entity_id))
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT * FROM metadata_cache_entities
|
||||
WHERE source = ? AND entity_type = ? AND entity_id = ?
|
||||
""", (source, entity_type, entity_id))
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
|
||||
# Touch
|
||||
cursor.execute("""
|
||||
UPDATE metadata_cache_entities
|
||||
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
|
||||
WHERE id = ?
|
||||
""", (row['id'],))
|
||||
conn.commit()
|
||||
|
||||
item = dict(row)
|
||||
# Parse JSON fields
|
||||
for json_field in ('genres', 'external_urls', 'raw_json'):
|
||||
if item.get(json_field):
|
||||
try:
|
||||
item[json_field] = json.loads(item[json_field])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
return item
|
||||
finally:
|
||||
conn.close()
|
||||
return None
|
||||
|
||||
# Touch
|
||||
cursor.execute("""
|
||||
UPDATE metadata_cache_entities
|
||||
SET last_accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1
|
||||
WHERE id = ?
|
||||
""", (row['id'],))
|
||||
conn.commit()
|
||||
|
||||
item = dict(row)
|
||||
# Parse JSON fields
|
||||
for json_field in ('genres', 'external_urls', 'raw_json'):
|
||||
if item.get(json_field):
|
||||
try:
|
||||
item[json_field] = json.loads(item[json_field])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
conn.close()
|
||||
return item
|
||||
except Exception as e:
|
||||
logger.error(f"Cache detail error: {e}")
|
||||
return None
|
||||
|
|
@ -420,49 +461,51 @@ class MetadataCache:
|
|||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
|
||||
stats = {
|
||||
'artists': {'spotify': 0, 'itunes': 0},
|
||||
'albums': {'spotify': 0, 'itunes': 0},
|
||||
'tracks': {'spotify': 0, 'itunes': 0},
|
||||
'searches': 0,
|
||||
'total_entries': 0,
|
||||
'total_hits': 0,
|
||||
'oldest': None,
|
||||
'newest': None,
|
||||
}
|
||||
stats = {
|
||||
'artists': {'spotify': 0, 'itunes': 0},
|
||||
'albums': {'spotify': 0, 'itunes': 0},
|
||||
'tracks': {'spotify': 0, 'itunes': 0},
|
||||
'searches': 0,
|
||||
'total_entries': 0,
|
||||
'total_hits': 0,
|
||||
'oldest': None,
|
||||
'newest': None,
|
||||
}
|
||||
|
||||
# Count by type and source (exclude pseudo-entities like _tracks, _features)
|
||||
cursor.execute(r"""
|
||||
SELECT entity_type, source, COUNT(*) as cnt, SUM(access_count) as hits
|
||||
FROM metadata_cache_entities
|
||||
WHERE entity_id NOT LIKE '%\_tracks' ESCAPE '\'
|
||||
AND entity_id NOT LIKE '%\_features' ESCAPE '\'
|
||||
GROUP BY entity_type, source
|
||||
""")
|
||||
type_key_map = {'artist': 'artists', 'album': 'albums', 'track': 'tracks'}
|
||||
for row in cursor.fetchall():
|
||||
et = type_key_map.get(row['entity_type'])
|
||||
src = row['source']
|
||||
if et and et in stats and src in stats[et]:
|
||||
stats[et][src] = row['cnt']
|
||||
stats['total_entries'] += row['cnt']
|
||||
stats['total_hits'] += (row['hits'] or 0)
|
||||
# Count by type and source (exclude pseudo-entities like _tracks, _features)
|
||||
cursor.execute(r"""
|
||||
SELECT entity_type, source, COUNT(*) as cnt, SUM(access_count) as hits
|
||||
FROM metadata_cache_entities
|
||||
WHERE entity_id NOT LIKE '%\_tracks' ESCAPE '\'
|
||||
AND entity_id NOT LIKE '%\_features' ESCAPE '\'
|
||||
GROUP BY entity_type, source
|
||||
""")
|
||||
type_key_map = {'artist': 'artists', 'album': 'albums', 'track': 'tracks'}
|
||||
for row in cursor.fetchall():
|
||||
et = type_key_map.get(row['entity_type'])
|
||||
src = row['source']
|
||||
if et and et in stats and src in stats[et]:
|
||||
stats[et][src] = row['cnt']
|
||||
stats['total_entries'] += row['cnt']
|
||||
stats['total_hits'] += (row['hits'] or 0)
|
||||
|
||||
# Search count
|
||||
cursor.execute("SELECT COUNT(*) as cnt FROM metadata_cache_searches")
|
||||
stats['searches'] = cursor.fetchone()['cnt']
|
||||
# Search count
|
||||
cursor.execute("SELECT COUNT(*) as cnt FROM metadata_cache_searches")
|
||||
stats['searches'] = cursor.fetchone()['cnt']
|
||||
|
||||
# Oldest and newest
|
||||
cursor.execute("SELECT MIN(created_at) as oldest, MAX(created_at) as newest FROM metadata_cache_entities")
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
stats['oldest'] = row['oldest']
|
||||
stats['newest'] = row['newest']
|
||||
# Oldest and newest
|
||||
cursor.execute("SELECT MIN(created_at) as oldest, MAX(created_at) as newest FROM metadata_cache_entities")
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
stats['oldest'] = row['oldest']
|
||||
stats['newest'] = row['newest']
|
||||
|
||||
conn.close()
|
||||
return stats
|
||||
return stats
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Cache stats error: {e}")
|
||||
return {
|
||||
|
|
@ -480,28 +523,30 @@ class MetadataCache:
|
|||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Entities
|
||||
cursor.execute("""
|
||||
DELETE FROM metadata_cache_entities
|
||||
WHERE julianday('now') - julianday(updated_at) > ttl_days
|
||||
""")
|
||||
entity_count = cursor.rowcount
|
||||
# Entities
|
||||
cursor.execute("""
|
||||
DELETE FROM metadata_cache_entities
|
||||
WHERE julianday('now') - julianday(updated_at) > ttl_days
|
||||
""")
|
||||
entity_count = cursor.rowcount
|
||||
|
||||
# Searches
|
||||
cursor.execute("""
|
||||
DELETE FROM metadata_cache_searches
|
||||
WHERE julianday('now') - julianday(created_at) > ttl_days
|
||||
""")
|
||||
search_count = cursor.rowcount
|
||||
# Searches
|
||||
cursor.execute("""
|
||||
DELETE FROM metadata_cache_searches
|
||||
WHERE julianday('now') - julianday(created_at) > ttl_days
|
||||
""")
|
||||
search_count = cursor.rowcount
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
total = entity_count + search_count
|
||||
if total > 0:
|
||||
logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)")
|
||||
return total
|
||||
conn.commit()
|
||||
total = entity_count + search_count
|
||||
if total > 0:
|
||||
logger.info(f"Evicted {total} expired cache entries ({entity_count} entities, {search_count} searches)")
|
||||
return total
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Cache eviction error: {e}")
|
||||
return 0
|
||||
|
|
@ -511,46 +556,48 @@ class MetadataCache:
|
|||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Clear entities
|
||||
where_parts = []
|
||||
params = []
|
||||
if source:
|
||||
where_parts.append('source = ?')
|
||||
params.append(source)
|
||||
if entity_type:
|
||||
where_parts.append('entity_type = ?')
|
||||
params.append(entity_type)
|
||||
# Clear entities
|
||||
where_parts = []
|
||||
params = []
|
||||
if source:
|
||||
where_parts.append('source = ?')
|
||||
params.append(source)
|
||||
if entity_type:
|
||||
where_parts.append('entity_type = ?')
|
||||
params.append(entity_type)
|
||||
|
||||
if where_parts:
|
||||
where_sql = ' AND '.join(where_parts)
|
||||
cursor.execute(f"DELETE FROM metadata_cache_entities WHERE {where_sql}", params)
|
||||
else:
|
||||
cursor.execute("DELETE FROM metadata_cache_entities")
|
||||
entity_count = cursor.rowcount
|
||||
if where_parts:
|
||||
where_sql = ' AND '.join(where_parts)
|
||||
cursor.execute(f"DELETE FROM metadata_cache_entities WHERE {where_sql}", params)
|
||||
else:
|
||||
cursor.execute("DELETE FROM metadata_cache_entities")
|
||||
entity_count = cursor.rowcount
|
||||
|
||||
# Clear searches (match source and entity_type → search_type)
|
||||
search_where = []
|
||||
search_params = []
|
||||
if source:
|
||||
search_where.append('source = ?')
|
||||
search_params.append(source)
|
||||
if entity_type:
|
||||
search_where.append('search_type = ?')
|
||||
search_params.append(entity_type)
|
||||
# Clear searches (match source and entity_type → search_type)
|
||||
search_where = []
|
||||
search_params = []
|
||||
if source:
|
||||
search_where.append('source = ?')
|
||||
search_params.append(source)
|
||||
if entity_type:
|
||||
search_where.append('search_type = ?')
|
||||
search_params.append(entity_type)
|
||||
|
||||
if search_where:
|
||||
cursor.execute(f"DELETE FROM metadata_cache_searches WHERE {' AND '.join(search_where)}", search_params)
|
||||
else:
|
||||
cursor.execute("DELETE FROM metadata_cache_searches")
|
||||
search_count = cursor.rowcount
|
||||
if search_where:
|
||||
cursor.execute(f"DELETE FROM metadata_cache_searches WHERE {' AND '.join(search_where)}", search_params)
|
||||
else:
|
||||
cursor.execute("DELETE FROM metadata_cache_searches")
|
||||
search_count = cursor.rowcount
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
total = entity_count + search_count
|
||||
logger.info(f"Cleared {total} cache entries (source={source}, type={entity_type})")
|
||||
return total
|
||||
conn.commit()
|
||||
total = entity_count + search_count
|
||||
logger.info(f"Cleared {total} cache entries (source={source}, type={entity_type})")
|
||||
return total
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Cache clear error: {e}")
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -1049,10 +1049,10 @@ class SpotifyClient:
|
|||
album = Album.from_spotify_album(album_data)
|
||||
albums.append(album)
|
||||
|
||||
# Cache individual albums + search mapping
|
||||
# Cache individual albums + search mapping (skip if full data already cached)
|
||||
entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')]
|
||||
if entries:
|
||||
cache.store_entities_bulk('spotify', 'album', entries)
|
||||
cache.store_entities_bulk('spotify', 'album', entries, skip_if_exists=True)
|
||||
cache.store_search_results('spotify', 'album', query, min(limit, 10),
|
||||
[ad.get('id') for ad in raw_items if ad.get('id')])
|
||||
|
||||
|
|
@ -1234,14 +1234,14 @@ class SpotifyClient:
|
|||
# Cache the aggregated result
|
||||
cache.store_entity('spotify', 'album', cache_key, result)
|
||||
|
||||
# Also cache individual tracks opportunistically
|
||||
# Also cache individual tracks opportunistically (skip if full data already cached)
|
||||
track_entries = []
|
||||
for track in all_tracks:
|
||||
tid = track.get('id')
|
||||
if tid:
|
||||
track_entries.append((tid, track))
|
||||
if track_entries:
|
||||
cache.store_entities_bulk('spotify', 'track', track_entries)
|
||||
cache.store_entities_bulk('spotify', 'track', track_entries, skip_if_exists=True)
|
||||
|
||||
return result
|
||||
|
||||
|
|
@ -1278,11 +1278,11 @@ class SpotifyClient:
|
|||
|
||||
logger.info(f"Retrieved {len(albums)} albums for artist {artist_id}")
|
||||
|
||||
# Cache individual albums opportunistically
|
||||
# Cache individual albums opportunistically (skip if full data already cached)
|
||||
cache = get_metadata_cache()
|
||||
entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')]
|
||||
if entries:
|
||||
cache.store_entities_bulk('spotify', 'album', entries)
|
||||
cache.store_entities_bulk('spotify', 'album', entries, skip_if_exists=True)
|
||||
|
||||
return albums
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue