diff --git a/core/database_update_worker.py b/core/database_update_worker.py index 54d0a818..9fbe0341 100644 --- a/core/database_update_worker.py +++ b/core/database_update_worker.py @@ -228,6 +228,20 @@ class DatabaseUpdateWorker(QThread): except Exception as e: logger.warning(f"Could not clear {self.server_type} cache: {e}") + # Detect and remove content deleted from the media server + if not self.full_refresh and self.database: + try: + removal_results = self._detect_and_remove_stale_content() + if removal_results: + r_artists = removal_results.get('artists_removed', 0) + r_albums = removal_results.get('albums_removed', 0) + r_tracks = removal_results.get('tracks_removed', 0) + if r_artists > 0 or r_albums > 0: + logger.info(f"🗑️ Removal detection: {r_artists} artists, " + f"{r_albums} albums, {r_tracks} tracks removed") + except Exception as e: + logger.warning(f"Removal detection failed (non-fatal): {e}") + # Cleanup orphaned records after incremental updates (catches fixed matches) if not self.full_refresh and self.database: try: @@ -252,15 +266,21 @@ class DatabaseUpdateWorker(QThread): except Exception as e: logger.warning(f"Could not merge duplicate artists: {e}") + # Store removal counts as instance attributes for the web server to access + removal = getattr(self, '_removal_results', None) + self.removed_artists = removal.get('artists_removed', 0) if removal else 0 + self.removed_albums = removal.get('albums_removed', 0) if removal else 0 + self.removed_tracks = removal.get('tracks_removed', 0) if removal else 0 + # Emit final results self._emit_signal('finished', self.processed_artists, - self.processed_albums, + self.processed_albums, self.processed_tracks, self.successful_operations, self.failed_operations ) - + update_type = "Full refresh" if self.full_refresh else "Incremental update" logger.info(f"{update_type} completed: {self.processed_artists} artists, " f"{self.processed_albums} albums, {self.processed_tracks} tracks processed") @@ -835,6 +855,113 @@ class DatabaseUpdateWorker(QThread): logger.debug(f"Error checking for metadata changes: {e}") return False # Assume no changes if we can't check + def _detect_and_remove_stale_content(self): + """Detect and remove content that was deleted from the media server. + + Compares the set of artist/album IDs in the database against what the + media server currently reports. Any IDs in the database but NOT on the + server are considered removed and are deleted. + + Includes safety checks to prevent accidental mass deletion if the server + returns suspiciously few results. + """ + self._emit_signal('phase_changed', "Checking for removed content...") + + # Check that the media client supports removal detection + if not hasattr(self.media_client, 'get_all_artist_ids') or not hasattr(self.media_client, 'get_all_album_ids'): + logger.info(f"Removal detection not supported for {self.server_type} — skipping") + return None + + # Fetch current IDs from media server (lightweight calls) + logger.info(f"🔍 Removal detection: fetching current IDs from {self.server_type}...") + server_artist_ids = self.media_client.get_all_artist_ids() + server_album_ids = self.media_client.get_all_album_ids() + + # Safety: if both come back empty, the server is unreachable + if not server_artist_ids and not server_album_ids: + logger.warning("🛡️ SAFETY: Server returned zero artists AND zero albums — " + "skipping removal detection") + return None + + # Get current DB counts for safety threshold + try: + db_stats = self.database.get_statistics_for_server(self.server_type) + db_artist_count = db_stats.get('artists', 0) + db_album_count = db_stats.get('albums', 0) + except Exception: + db_artist_count = 0 + db_album_count = 0 + + # Per-type safety: skip removal for any type where the server returned + # empty or suspiciously few results. An empty set means the API call + # failed, not that the server has zero items. + check_artists = bool(server_artist_ids) + check_albums = bool(server_album_ids) + + if check_artists and db_artist_count > 100: + if len(server_artist_ids) < db_artist_count * 0.5: + logger.warning( + f"🛡️ SAFETY: Server reported {len(server_artist_ids)} artists but " + f"database has {db_artist_count} — skipping artist removal check") + check_artists = False + + if check_albums and db_album_count > 100: + if len(server_album_ids) < db_album_count * 0.5: + logger.warning( + f"🛡️ SAFETY: Server reported {len(server_album_ids)} albums but " + f"database has {db_album_count} — skipping album removal check") + check_albums = False + + if not check_artists and not check_albums: + logger.warning("🛡️ SAFETY: Both artist and album checks disabled — " + "skipping removal detection") + return None + + # Get stored IDs from database + db_artist_ids = self.database.get_all_artist_ids_for_server(self.server_type) if check_artists else set() + db_album_ids = self.database.get_all_album_ids_for_server(self.server_type) if check_albums else set() + + # Compute removal sets (only for types we have valid server data for) + removed_artist_ids = (db_artist_ids - server_artist_ids) if check_artists else set() + removed_album_ids = (db_album_ids - server_album_ids) if check_albums else set() + + # Filter out albums that will already be cascade-deleted with their artist + if removed_artist_ids and removed_album_ids: + try: + with self.database._get_connection() as conn: + cursor = conn.cursor() + artist_list = list(removed_artist_ids) + cascade_album_ids = set() + batch_size = 500 + for i in range(0, len(artist_list), batch_size): + batch = artist_list[i:i + batch_size] + placeholders = ','.join('?' * len(batch)) + cursor.execute( + f"SELECT id FROM albums WHERE artist_id IN ({placeholders}) " + f"AND server_source = ?", + batch + [self.server_type]) + cascade_album_ids.update(row[0] for row in cursor.fetchall()) + removed_album_ids -= cascade_album_ids + except Exception: + pass # If this optimization fails, double-delete is harmless + + if not removed_artist_ids and not removed_album_ids: + logger.info("🔍 Removal detection: no stale content found") + return {'artists_removed': 0, 'albums_removed': 0, 'tracks_removed': 0} + + logger.info(f"🗑️ Removal detection: found {len(removed_artist_ids)} removed artists, " + f"{len(removed_album_ids)} removed albums") + + self._emit_signal('phase_changed', + f"Removing {len(removed_artist_ids)} artists, " + f"{len(removed_album_ids)} albums no longer on server...") + + results = self.database.delete_removed_content( + removed_artist_ids, removed_album_ids, self.server_type) + + self._removal_results = results + return results + def _get_recent_albums_for_server(self) -> List: """Get recently added albums using server-specific methods""" try: diff --git a/core/jellyfin_client.py b/core/jellyfin_client.py index e8c54b59..05cd9531 100644 --- a/core/jellyfin_client.py +++ b/core/jellyfin_client.py @@ -740,6 +740,65 @@ class JellyfinClient: logger.error(f"Error getting artists from Jellyfin: {e}") return [] + def get_all_artist_ids(self) -> set: + """Get all artist IDs from Jellyfin (lightweight, for removal detection). + Does NOT trigger _populate_aggressive_cache().""" + if not self.ensure_connection() or not self.music_library_id: + return set() + try: + params = { + 'ParentId': self.music_library_id, + 'Recursive': True, + 'Fields': '', + 'EnableTotalRecordCount': False + } + response = self._make_request('/Artists/AlbumArtists', params) + if not response: + return set() + ids = {item['Id'] for item in response.get('Items', []) if 'Id' in item} + logger.info(f"Retrieved {len(ids)} artist IDs from Jellyfin (lightweight)") + return ids + except Exception as e: + logger.error(f"Error getting artist IDs from Jellyfin: {e}") + return set() + + def get_all_album_ids(self) -> set: + """Get all album IDs from Jellyfin (lightweight, paginated, for removal detection). + Does NOT trigger _populate_aggressive_cache().""" + if not self.ensure_connection() or not self.music_library_id: + return set() + try: + all_ids = set() + start_index = 0 + limit = 10000 + while True: + params = { + 'ParentId': self.music_library_id, + 'IncludeItemTypes': 'MusicAlbum', + 'Recursive': True, + 'Fields': '', + 'SortBy': 'SortName', + 'SortOrder': 'Ascending', + 'StartIndex': start_index, + 'Limit': limit, + 'EnableTotalRecordCount': False + } + response = self._make_request(f'/Users/{self.user_id}/Items', params) + if not response: + break + items = response.get('Items', []) + if not items: + break + all_ids.update(item['Id'] for item in items if 'Id' in item) + if len(items) < limit: + break + start_index += limit + logger.info(f"Retrieved {len(all_ids)} album IDs from Jellyfin (lightweight)") + return all_ids + except Exception as e: + logger.error(f"Error getting album IDs from Jellyfin: {e}") + return set() + def get_albums_for_artist(self, artist_id: str) -> List[JellyfinAlbum]: """Get all albums for a specific artist""" # Use cache if available diff --git a/core/navidrome_client.py b/core/navidrome_client.py index 7a14cd89..4f8a07eb 100644 --- a/core/navidrome_client.py +++ b/core/navidrome_client.py @@ -390,6 +390,64 @@ class NavidromeClient: logger.error(f"Error getting artists from Navidrome: {e}") return [] + def get_all_artist_ids(self) -> set: + """Get all artist IDs from Navidrome (lightweight, for removal detection).""" + if not self.ensure_connection(): + return set() + try: + params = {} + if self.music_folder_id: + params['musicFolderId'] = self.music_folder_id + response = self._make_request('getArtists', params if params else None) + if not response: + return set() + ids = set() + for index in response.get('artists', {}).get('index', []): + for artist_data in index.get('artist', []): + aid = artist_data.get('id') + if aid: + ids.add(str(aid)) + logger.info(f"Retrieved {len(ids)} artist IDs from Navidrome (lightweight)") + return ids + except Exception as e: + logger.error(f"Error getting artist IDs from Navidrome: {e}") + return set() + + def get_all_album_ids(self) -> set: + """Get all album IDs from Navidrome (lightweight, paginated, for removal detection).""" + if not self.ensure_connection(): + return set() + try: + all_ids = set() + page_size = 500 + offset = 0 + while True: + params = { + 'type': 'alphabeticalByName', + 'size': page_size, + 'offset': offset + } + if self.music_folder_id: + params['musicFolderId'] = self.music_folder_id + response = self._make_request('getAlbumList2', params) + if not response: + break + album_list = response.get('albumList2', {}).get('album', []) + if not album_list: + break + for album_data in album_list: + aid = album_data.get('id') + if aid: + all_ids.add(str(aid)) + if len(album_list) < page_size: + break + offset += page_size + logger.info(f"Retrieved {len(all_ids)} album IDs from Navidrome (lightweight)") + return all_ids + except Exception as e: + logger.error(f"Error getting album IDs from Navidrome: {e}") + return set() + def get_albums_for_artist(self, artist_id: str) -> List[NavidromeAlbum]: """Get all albums for a specific artist""" # Check cache first diff --git a/core/plex_client.py b/core/plex_client.py index afd7db35..26f6fbdf 100644 --- a/core/plex_client.py +++ b/core/plex_client.py @@ -617,7 +617,7 @@ class PlexClient: if not self.ensure_connection() or not self.music_library: logger.error("Not connected to Plex server or no music library") return [] - + try: artists = self.music_library.searchArtists() logger.info(f"Found {len(artists)} artists in Plex library") @@ -625,7 +625,33 @@ class PlexClient: except Exception as e: logger.error(f"Error getting all artists: {e}") return [] - + + def get_all_artist_ids(self) -> set: + """Get all artist IDs from Plex library (lightweight, for removal detection).""" + if not self.ensure_connection() or not self.music_library: + return set() + try: + artists = self.music_library.searchArtists() + ids = {str(a.ratingKey) for a in artists} + logger.info(f"Retrieved {len(ids)} artist IDs from Plex") + return ids + except Exception as e: + logger.error(f"Error getting artist IDs from Plex: {e}") + return set() + + def get_all_album_ids(self) -> set: + """Get all album IDs from Plex library (lightweight, for removal detection).""" + if not self.ensure_connection() or not self.music_library: + return set() + try: + albums = self.music_library.albums() + ids = {str(a.ratingKey) for a in albums} + logger.info(f"Retrieved {len(ids)} album IDs from Plex") + return ids + except Exception as e: + logger.error(f"Error getting album IDs from Plex: {e}") + return set() + def update_artist_genres(self, artist: PlexArtist, genres: List[str]): """Update artist genres""" try: diff --git a/database/music_database.py b/database/music_database.py index 06d3839c..d6e8d30d 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -1615,6 +1615,115 @@ class MusicDatabase: logger.error(f"Error merging duplicate artists: {e}") return {'artists_merged': 0, 'albums_migrated': 0} + # --- Removal detection helpers --- + + def get_all_artist_ids_for_server(self, server_source: str) -> set: + """Get all artist IDs stored in the database for a specific server.""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT id FROM artists WHERE server_source = ?", (server_source,)) + return {row[0] for row in cursor.fetchall()} + except Exception as e: + logger.error(f"Error getting artist IDs for {server_source}: {e}") + return set() + + def get_all_album_ids_for_server(self, server_source: str) -> set: + """Get all album IDs stored in the database for a specific server.""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT id FROM albums WHERE server_source = ?", (server_source,)) + return {row[0] for row in cursor.fetchall()} + except Exception as e: + logger.error(f"Error getting album IDs for {server_source}: {e}") + return set() + + def delete_removed_content(self, removed_artist_ids: set, removed_album_ids: set, + server_source: str): + """Delete artists and albums that were removed from the media server. + Manually cascades deletes (tracks -> albums -> artists) to match existing patterns.""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + batch_size = 500 + + artists_removed = 0 + albums_removed = 0 + tracks_removed = 0 + + # Remove artists and their children + if removed_artist_ids: + artist_list = list(removed_artist_ids) + for i in range(0, len(artist_list), batch_size): + batch = artist_list[i:i + batch_size] + placeholders = ','.join('?' * len(batch)) + params = batch + [server_source] + + # Delete tracks belonging to these artists + cursor.execute( + f"SELECT COUNT(*) FROM tracks WHERE artist_id IN ({placeholders}) AND server_source = ?", + params) + tracks_removed += cursor.fetchone()[0] + cursor.execute( + f"DELETE FROM tracks WHERE artist_id IN ({placeholders}) AND server_source = ?", + params) + + # Delete albums belonging to these artists + cursor.execute( + f"SELECT COUNT(*) FROM albums WHERE artist_id IN ({placeholders}) AND server_source = ?", + params) + albums_removed += cursor.fetchone()[0] + cursor.execute( + f"DELETE FROM albums WHERE artist_id IN ({placeholders}) AND server_source = ?", + params) + + # Delete the artists themselves + cursor.execute( + f"DELETE FROM artists WHERE id IN ({placeholders}) AND server_source = ?", + params) + artists_removed += cursor.rowcount + + # Remove albums (not already handled by artist cascade above) + if removed_album_ids: + album_list = list(removed_album_ids) + for i in range(0, len(album_list), batch_size): + batch = album_list[i:i + batch_size] + placeholders = ','.join('?' * len(batch)) + params = batch + [server_source] + + # Delete tracks belonging to these albums + cursor.execute( + f"SELECT COUNT(*) FROM tracks WHERE album_id IN ({placeholders}) AND server_source = ?", + params) + tracks_removed += cursor.fetchone()[0] + cursor.execute( + f"DELETE FROM tracks WHERE album_id IN ({placeholders}) AND server_source = ?", + params) + + # Delete the albums themselves + cursor.execute( + f"DELETE FROM albums WHERE id IN ({placeholders}) AND server_source = ?", + params) + albums_removed += cursor.rowcount + + conn.commit() + + if artists_removed > 0 or albums_removed > 0: + logger.info(f"Removal cleanup for {server_source}: " + f"{artists_removed} artists, {albums_removed} albums, " + f"{tracks_removed} tracks removed") + + return { + 'artists_removed': artists_removed, + 'albums_removed': albums_removed, + 'tracks_removed': tracks_removed + } + + except Exception as e: + logger.error(f"Error deleting removed content for {server_source}: {e}") + return {'artists_removed': 0, 'albums_removed': 0, 'tracks_removed': 0} + # Artist operations def insert_or_update_artist(self, plex_artist) -> bool: """Insert or update artist from Plex artist object - DEPRECATED: Use insert_or_update_media_artist instead""" diff --git a/web_server.py b/web_server.py index ccf0f47a..a9b8ee5b 100644 --- a/web_server.py +++ b/web_server.py @@ -247,7 +247,10 @@ db_update_state = { "current_item": "", "processed": 0, "total": 0, - "error_message": "" + "error_message": "", + "removed_artists": 0, + "removed_albums": 0, + "removed_tracks": 0 } # Quality Scanner state @@ -12626,12 +12629,30 @@ def _db_update_phase_callback(phase): db_update_state["phase"] = phase def _db_update_finished_callback(total_artists, total_albums, total_tracks, successful, failed): + # Check for removal results from the worker + removed_artists = 0 + removed_albums = 0 + removed_tracks = 0 + if db_update_worker: + removed_artists = getattr(db_update_worker, 'removed_artists', 0) + removed_albums = getattr(db_update_worker, 'removed_albums', 0) + removed_tracks = getattr(db_update_worker, 'removed_tracks', 0) + + removal_msg = "" + if removed_artists > 0 or removed_albums > 0: + removal_msg = f" | Removed: {removed_artists} artists, {removed_albums} albums" + with db_update_lock: db_update_state["status"] = "finished" - db_update_state["phase"] = f"Completed: {successful} successful, {failed} failed." - + db_update_state["phase"] = f"Completed: {successful} successful, {failed} failed{removal_msg}." + db_update_state["removed_artists"] = removed_artists + db_update_state["removed_albums"] = removed_albums + db_update_state["removed_tracks"] = removed_tracks + # Add activity for database update completion summary = f"{total_tracks} tracks, {total_albums} albums, {total_artists} artists processed" + if removed_artists > 0 or removed_albums > 0: + summary += f" | {removed_artists} artists, {removed_albums} albums removed" add_activity_item("✅", "Database Update Complete", summary, "Now") # WISHLIST CLEANUP: Automatically clean up wishlist after database update