From d264ec70f3d6685db6338c2291493611428d42b8 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Fri, 6 Mar 2026 09:41:44 -0800 Subject: [PATCH] Add sync match cache and fix discovery clear to purge cache --- database/music_database.py | 121 +++++++++++++++++++++++++++++++++++++ services/sync_service.py | 63 +++++++++++++++++-- web_server.py | 89 ++++++++++++++++++++++++--- webui/static/script.js | 30 +++++++++ webui/static/style.css | 38 ++++++++++++ 5 files changed, 328 insertions(+), 13 deletions(-) diff --git a/database/music_database.py b/database/music_database.py index 80682cb5..11dde045 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -896,6 +896,25 @@ class MusicDatabase: """) cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_cache_lookup ON discovery_match_cache (normalized_title, normalized_artist, provider)") + # Sync match cache — caches server track ID for discovered Spotify tracks + cursor.execute(""" + CREATE TABLE IF NOT EXISTS sync_match_cache ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + spotify_track_id TEXT NOT NULL, + normalized_title TEXT NOT NULL, + normalized_artist TEXT NOT NULL, + server_source TEXT NOT NULL, + server_track_id INTEGER NOT NULL, + server_track_title TEXT, + confidence REAL NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + last_used_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + use_count INTEGER DEFAULT 1, + UNIQUE(spotify_track_id, server_source) + ) + """) + cursor.execute("CREATE INDEX IF NOT EXISTS idx_sync_cache_lookup ON sync_match_cache (spotify_track_id, server_source)") + logger.info("Discovery tables created successfully") except Exception as e: @@ -5994,6 +6013,73 @@ class MusicDatabase: logger.error(f"Error saving discovery cache: {e}") return False + # ==================== Sync Match Cache ==================== + + def read_sync_match_cache(self, spotify_track_id: str, server_source: str) -> Optional[Dict]: + """Read a cached sync match. Returns {server_track_id, server_track_title, confidence} or None. + Also bumps last_used_at and use_count on hit.""" + try: + conn = self._get_connection() + cursor = conn.cursor() + cursor.execute(""" + SELECT server_track_id, server_track_title, confidence FROM sync_match_cache + WHERE spotify_track_id = ? AND server_source = ? + """, (spotify_track_id, server_source)) + row = cursor.fetchone() + if row: + cursor.execute(""" + UPDATE sync_match_cache + SET last_used_at = CURRENT_TIMESTAMP, use_count = use_count + 1 + WHERE spotify_track_id = ? AND server_source = ? + """, (spotify_track_id, server_source)) + conn.commit() + return { + 'server_track_id': row['server_track_id'], + 'server_track_title': row['server_track_title'], + 'confidence': row['confidence'], + } + return None + except Exception as e: + logger.error(f"Error reading sync match cache: {e}") + return None + + def save_sync_match_cache(self, spotify_track_id: str, normalized_title: str, + normalized_artist: str, server_source: str, + server_track_id, server_track_title: str, + confidence: float) -> bool: + """Save a sync match to cache. Uses INSERT OR REPLACE for upsert.""" + try: + conn = self._get_connection() + cursor = conn.cursor() + cursor.execute(""" + INSERT OR REPLACE INTO sync_match_cache + (spotify_track_id, normalized_title, normalized_artist, server_source, + server_track_id, server_track_title, confidence, + created_at, last_used_at, use_count) + VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 1) + """, (spotify_track_id, normalized_title, normalized_artist, server_source, + server_track_id, server_track_title, confidence)) + conn.commit() + return True + except Exception as e: + logger.error(f"Error saving sync match cache: {e}") + return False + + def invalidate_sync_match_cache(self, server_source: str = None) -> int: + """Clear sync match cache entries. If server_source given, only clear that server's entries.""" + try: + conn = self._get_connection() + cursor = conn.cursor() + if server_source: + cursor.execute("DELETE FROM sync_match_cache WHERE server_source = ?", (server_source,)) + else: + cursor.execute("DELETE FROM sync_match_cache") + conn.commit() + return cursor.rowcount + except Exception as e: + logger.error(f"Error invalidating sync match cache: {e}") + return 0 + # ==================== Retag Tool Methods ==================== def add_retag_group(self, group_type: str, artist_name: str, album_name: str, @@ -6564,6 +6650,41 @@ class MusicDatabase: logger.error(f"Error getting extra_data map: {e}") return {} + def clear_mirrored_playlist_discovery(self, playlist_id: int) -> int: + """Clear extra_data for all tracks in a mirrored playlist (resets discovery).""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + "UPDATE mirrored_playlist_tracks SET extra_data = NULL WHERE playlist_id = ?", + (playlist_id,) + ) + conn.commit() + return cursor.rowcount + except Exception as e: + logger.error(f"Error clearing mirrored playlist discovery: {e}") + return 0 + + def get_mirrored_playlist_discovery_counts(self, playlist_id: int) -> tuple: + """Return (discovered_count, total_count) for a mirrored playlist.""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT COUNT(*) as total FROM mirrored_playlist_tracks WHERE playlist_id = ?", + (playlist_id,) + ) + total = cursor.fetchone()['total'] + cursor.execute( + "SELECT COUNT(*) as discovered FROM mirrored_playlist_tracks WHERE playlist_id = ? AND extra_data LIKE '%\"discovered\": true%'", + (playlist_id,) + ) + discovered = cursor.fetchone()['discovered'] + return (discovered, total) + except Exception as e: + logger.error(f"Error getting mirrored playlist discovery counts: {e}") + return (0, 0) + def delete_mirrored_playlist(self, playlist_id: int) -> bool: """Delete a mirrored playlist and its tracks (CASCADE).""" try: diff --git a/services/sync_service.py b/services/sync_service.py index 70279fd3..74e9095f 100644 --- a/services/sync_service.py +++ b/services/sync_service.py @@ -351,9 +351,52 @@ class PlaylistSyncService: # Use the SAME improved database matching as PlaylistTrackAnalysisWorker from database.music_database import MusicDatabase - + from config.settings import config_manager + original_title = spotify_track.name - + spotify_id = getattr(spotify_track, 'id', '') or '' + active_server = config_manager.get_active_media_server() + + # --- Sync match cache fast-path --- + if spotify_id: + try: + cache_db = MusicDatabase() + cached = cache_db.read_sync_match_cache(spotify_id, active_server) + if cached: + server_track_id = cached['server_track_id'] + db_track_check = cache_db.get_track_by_id(server_track_id) + if db_track_check: + if server_type == "jellyfin": + class JellyfinTrackFromCache: + def __init__(self, db_t): + self.ratingKey = db_t.id + self.title = db_t.title + self.id = db_t.id + actual_track = JellyfinTrackFromCache(db_track_check) + elif server_type == "navidrome": + class NavidromeTrackFromCache: + def __init__(self, db_t): + self.ratingKey = db_t.id + self.title = db_t.title + self.id = db_t.id + actual_track = NavidromeTrackFromCache(db_track_check) + else: + try: + actual_track = media_client.server.fetchItem(int(server_track_id)) + if not (actual_track and hasattr(actual_track, 'ratingKey')): + actual_track = None + except Exception: + actual_track = None + + if actual_track: + logger.debug(f"⚡ Sync cache hit: '{original_title}' → server track {server_track_id}") + return actual_track, cached['confidence'] + + logger.debug(f"🔄 Sync cache stale for '{original_title}' — track {server_track_id} gone") + except Exception as cache_err: + logger.debug(f"Sync cache lookup error: {cache_err}") + # --- End cache fast-path --- + # Try each artist (same as modal logic) for artist in spotify_track.artists: if self._cancelled: @@ -369,13 +412,23 @@ class PlaylistSyncService: # Use the improved database check_track_exists method with server awareness try: - from config.settings import config_manager - active_server = config_manager.get_active_media_server() db = MusicDatabase() db_track, confidence = db.check_track_exists(original_title, artist_name, confidence_threshold=0.7, server_source=active_server) - + if db_track and confidence >= 0.7: logger.debug(f"✔️ Database match found for '{original_title}' by '{artist_name}': '{db_track.title}' with confidence {confidence:.2f}") + + # Save to sync match cache for next time + if spotify_id: + try: + from core.matching_engine import MusicMatchingEngine + me = MusicMatchingEngine() + db.save_sync_match_cache( + spotify_id, me.clean_title(original_title), me.clean_artist(artist_name), + active_server, db_track.id, db_track.title, confidence + ) + except Exception: + pass # Fetch the actual track object from active media server using the database track ID try: diff --git a/web_server.py b/web_server.py index e2af623d..e85af743 100644 --- a/web_server.py +++ b/web_server.py @@ -13704,6 +13704,15 @@ def _db_update_finished_callback(total_artists, total_albums, total_tracks, succ except Exception: pass + # Invalidate sync match cache (track IDs may have changed) + try: + inv_db = get_database() + cleared = inv_db.invalidate_sync_match_cache() + if cleared: + logger.info(f"🗑️ Cleared {cleared} sync match cache entries after database update") + except Exception: + pass + # WISHLIST CLEANUP: Automatically clean up wishlist after database update try: print("📋 [DB Update] Database update completed, starting automatic wishlist cleanup...") @@ -22151,11 +22160,31 @@ def _run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None): try: from database.music_database import MusicDatabase from config.settings import config_manager - + db = MusicDatabase() active_server = config_manager.get_active_media_server() original_title = spotify_track.name - + spotify_id = getattr(spotify_track, 'id', '') or '' + + # --- Sync match cache fast-path --- + if spotify_id: + try: + cached = db.read_sync_match_cache(spotify_id, active_server) + if cached: + db_track_check = db.get_track_by_id(cached['server_track_id']) + if db_track_check: + class DatabaseTrackCached: + def __init__(self, db_t): + self.ratingKey = db_t.id + self.title = db_t.title + self.id = db_t.id + print(f"⚡ Sync cache hit: '{original_title}' → server track {cached['server_track_id']}") + return DatabaseTrackCached(db_track_check), cached['confidence'] + print(f"🔄 Sync cache stale for '{original_title}' — track gone") + except Exception: + pass + # --- End cache fast-path --- + # Try each artist (same logic as original) for artist in spotify_track.artists: # Extract artist name from both string and dict formats @@ -22165,7 +22194,7 @@ def _run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None): artist_name = artist['name'] else: artist_name = str(artist) - + db_track, confidence = db.check_track_exists( original_title, artist_name, confidence_threshold=0.80, @@ -22174,20 +22203,31 @@ def _run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None): if db_track and confidence >= 0.80: print(f"✅ Database match: '{db_track.title}' (confidence: {confidence:.2f})") - + + # Save to sync match cache + if spotify_id: + try: + from core.matching_engine import MusicMatchingEngine + me = MusicMatchingEngine() + db.save_sync_match_cache( + spotify_id, me.clean_title(original_title), me.clean_artist(artist_name), + active_server, db_track.id, db_track.title, confidence + ) + except Exception: + pass + # Create mock track object for playlist creation class DatabaseTrackMock: def __init__(self, db_track): self.ratingKey = db_track.id self.title = db_track.title self.id = db_track.id - # Add any other attributes needed for playlist creation - + return DatabaseTrackMock(db_track), confidence - + print(f"❌ No database match found for: '{original_title}'") return None, 0.0 - + except Exception as e: print(f"❌ Database search error: {e}") return None, 0.0 @@ -29028,6 +29068,10 @@ def get_mirrored_playlists_endpoint(): database = get_database() profile_id = get_current_profile_id() playlists = database.get_mirrored_playlists(profile_id=profile_id) + for pl in playlists: + discovered, total = database.get_mirrored_playlist_discovery_counts(pl['id']) + pl['discovered_count'] = discovered + pl['total_count'] = total return jsonify(playlists) except Exception as e: logger.error(f"Error getting mirrored playlists: {e}") @@ -29059,6 +29103,35 @@ def delete_mirrored_playlist_endpoint(playlist_id): logger.error(f"Error deleting mirrored playlist: {e}") return jsonify({"error": str(e)}), 500 +@app.route('/api/mirrored-playlists//clear-discovery', methods=['POST']) +def clear_mirrored_discovery_endpoint(playlist_id): + """Clear discovery data for all tracks in a mirrored playlist, including discovery cache.""" + try: + database = get_database() + + # Clear discovery cache entries for these tracks so re-discovery does fresh lookups + try: + tracks = database.get_mirrored_playlist_tracks(playlist_id) + if tracks: + conn = database._get_connection() + cursor = conn.cursor() + for t in tracks: + cache_key = _get_discovery_cache_key(t.get('track_name', ''), t.get('artist_name', '')) + cursor.execute( + "DELETE FROM discovery_match_cache WHERE normalized_title = ? AND normalized_artist = ?", + (cache_key[0], cache_key[1]) + ) + conn.commit() + logger.info(f"Cleared discovery cache for {len(tracks)} tracks in playlist {playlist_id}") + except Exception as cache_err: + logger.warning(f"Error clearing discovery cache: {cache_err}") + + cleared = database.clear_mirrored_playlist_discovery(playlist_id) + return jsonify({"success": True, "cleared": cleared}) + except Exception as e: + logger.error(f"Error clearing mirrored discovery: {e}") + return jsonify({"error": str(e)}), 500 + @app.route('/api/mirrored-playlists//prepare-discovery', methods=['POST']) def prepare_mirrored_discovery(playlist_id): """Register a mirrored playlist into youtube_playlist_states so the YouTube discovery pipeline can run.""" diff --git a/webui/static/script.js b/webui/static/script.js index d60f1c3f..2dc1cab6 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -41671,6 +41671,15 @@ function renderMirroredCard(p, container) { const sourceIcons = { spotify: '🎵', tidal: '🌊', youtube: '▶', beatport: '🎛' }; const srcIcon = sourceIcons[p.source] || '📋'; + // Discovery ratio + const disc = p.discovered_count || 0; + const tot = p.total_count || p.track_count || 0; + let ratioHtml = ''; + if (disc > 0) { + const complete = disc >= tot; + ratioHtml = `${disc}/${tot} discovered`; + } + const card = document.createElement('div'); card.className = 'mirrored-playlist-card'; card.id = `mirrored-card-${p.id}`; @@ -41682,9 +41691,11 @@ function renderMirroredCard(p, container) { ${_esc(p.source)} ${p.track_count} tracks Mirrored ${ago} + ${ratioHtml} ${phaseHtml} + ${disc > 0 ? `` : ''} `; card.addEventListener('click', () => { @@ -42035,6 +42046,25 @@ function closeMirroredModal() { /** * Delete a mirrored playlist after confirmation. */ +async function clearMirroredDiscovery(playlistId, name) { + if (!confirm(`Clear discovery data for "${name}"? You can re-discover afterwards to get updated cover art.`)) return; + try { + const res = await fetch(`/api/mirrored-playlists/${playlistId}/clear-discovery`, { method: 'POST' }); + const data = await res.json(); + if (data.success) { + showToast(`Cleared discovery for ${name} (${data.cleared} tracks)`, 'success'); + // Also clear the discovery state so the card goes back to fresh + const hash = `mirrored_${playlistId}`; + delete youtubePlaylistStates[hash]; + loadMirroredPlaylists(); + } else { + showToast(data.error || 'Failed to clear discovery', 'error'); + } + } catch (err) { + showToast(`Error: ${err.message}`, 'error'); + } +} + async function deleteMirroredPlaylist(playlistId, name) { if (!confirm(`Delete mirrored playlist "${name}"?`)) return; try { diff --git a/webui/static/style.css b/webui/static/style.css index f3a3a519..b2931404 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -5972,6 +5972,44 @@ body { transform: scale(1.1); } +.mirrored-card-clear { + background: rgba(255, 255, 255, 0.06); + border: 1px solid rgba(255, 255, 255, 0.08); + color: #555; + cursor: pointer; + font-size: 14px; + padding: 0; + width: 32px; + height: 32px; + display: flex; + align-items: center; + justify-content: center; + border-radius: 50%; + transition: all 0.2s ease; + flex-shrink: 0; + opacity: 0; +} + +.mirrored-playlist-card:hover .mirrored-card-clear { + opacity: 1; +} + +.mirrored-card-clear:hover { + color: #a78bfa; + background: rgba(167, 139, 250, 0.15); + border-color: rgba(167, 139, 250, 0.3); + transform: scale(1.1); +} + +.discovery-ratio { + font-size: 0.8em; + color: rgba(255, 255, 255, 0.4); +} + +.discovery-ratio.complete { + color: #22c55e; +} + /* Mirrored playlist track modal */ .mirrored-modal-overlay { position: fixed;