Detect and remove deleted content during incremental database updates

Incremental database updates now detect when artists or albums have been removed from your media server (Plex, Jellyfin, or Navidrome) and automatically clean them up from SoulSync's database. Previously, deleted content would persist as ghost entries until you ran a full refresh. Removal counts are reported in the scan results. Includes safety checks to prevent accidental mass deletion if the server is unreachable or returns incomplete data.

core/database_update_worker.py

@@ -228,6 +228,20 @@ class DatabaseUpdateWorker(QThread):
                 except Exception as e:
                     logger.warning(f"Could not clear {self.server_type} cache: {e}")
             
+            # Detect and remove content deleted from the media server
+            if not self.full_refresh and self.database:
+                try:
+                    removal_results = self._detect_and_remove_stale_content()
+                    if removal_results:
+                        r_artists = removal_results.get('artists_removed', 0)
+                        r_albums = removal_results.get('albums_removed', 0)
+                        r_tracks = removal_results.get('tracks_removed', 0)
+                        if r_artists > 0 or r_albums > 0:
+                            logger.info(f"🗑️ Removal detection: {r_artists} artists, "
+                                       f"{r_albums} albums, {r_tracks} tracks removed")
+                except Exception as e:
+                    logger.warning(f"Removal detection failed (non-fatal): {e}")
+
             # Cleanup orphaned records after incremental updates (catches fixed matches)
             if not self.full_refresh and self.database:
                 try:
@@ -252,15 +266,21 @@ class DatabaseUpdateWorker(QThread):
                 except Exception as e:
                     logger.warning(f"Could not merge duplicate artists: {e}")
             
+            # Store removal counts as instance attributes for the web server to access
+            removal = getattr(self, '_removal_results', None)
+            self.removed_artists = removal.get('artists_removed', 0) if removal else 0
+            self.removed_albums = removal.get('albums_removed', 0) if removal else 0
+            self.removed_tracks = removal.get('tracks_removed', 0) if removal else 0
+
             # Emit final results
             self._emit_signal('finished',
                 self.processed_artists,
-                self.processed_albums, 
+                self.processed_albums,
                 self.processed_tracks,
                 self.successful_operations,
                 self.failed_operations
             )
-            
+
             update_type = "Full refresh" if self.full_refresh else "Incremental update"
             logger.info(f"{update_type} completed: {self.processed_artists} artists, "
                        f"{self.processed_albums} albums, {self.processed_tracks} tracks processed")
@@ -835,6 +855,113 @@ class DatabaseUpdateWorker(QThread):
             logger.debug(f"Error checking for metadata changes: {e}")
             return False  # Assume no changes if we can't check
     
+    def _detect_and_remove_stale_content(self):
+        """Detect and remove content that was deleted from the media server.
+
+        Compares the set of artist/album IDs in the database against what the
+        media server currently reports. Any IDs in the database but NOT on the
+        server are considered removed and are deleted.
+
+        Includes safety checks to prevent accidental mass deletion if the server
+        returns suspiciously few results.
+        """
+        self._emit_signal('phase_changed', "Checking for removed content...")
+
+        # Check that the media client supports removal detection
+        if not hasattr(self.media_client, 'get_all_artist_ids') or not hasattr(self.media_client, 'get_all_album_ids'):
+            logger.info(f"Removal detection not supported for {self.server_type} — skipping")
+            return None
+
+        # Fetch current IDs from media server (lightweight calls)
+        logger.info(f"🔍 Removal detection: fetching current IDs from {self.server_type}...")
+        server_artist_ids = self.media_client.get_all_artist_ids()
+        server_album_ids = self.media_client.get_all_album_ids()
+
+        # Safety: if both come back empty, the server is unreachable
+        if not server_artist_ids and not server_album_ids:
+            logger.warning("🛡️ SAFETY: Server returned zero artists AND zero albums — "
+                          "skipping removal detection")
+            return None
+
+        # Get current DB counts for safety threshold
+        try:
+            db_stats = self.database.get_statistics_for_server(self.server_type)
+            db_artist_count = db_stats.get('artists', 0)
+            db_album_count = db_stats.get('albums', 0)
+        except Exception:
+            db_artist_count = 0
+            db_album_count = 0
+
+        # Per-type safety: skip removal for any type where the server returned
+        # empty or suspiciously few results. An empty set means the API call
+        # failed, not that the server has zero items.
+        check_artists = bool(server_artist_ids)
+        check_albums = bool(server_album_ids)
+
+        if check_artists and db_artist_count > 100:
+            if len(server_artist_ids) < db_artist_count * 0.5:
+                logger.warning(
+                    f"🛡️ SAFETY: Server reported {len(server_artist_ids)} artists but "
+                    f"database has {db_artist_count} — skipping artist removal check")
+                check_artists = False
+
+        if check_albums and db_album_count > 100:
+            if len(server_album_ids) < db_album_count * 0.5:
+                logger.warning(
+                    f"🛡️ SAFETY: Server reported {len(server_album_ids)} albums but "
+                    f"database has {db_album_count} — skipping album removal check")
+                check_albums = False
+
+        if not check_artists and not check_albums:
+            logger.warning("🛡️ SAFETY: Both artist and album checks disabled — "
+                          "skipping removal detection")
+            return None
+
+        # Get stored IDs from database
+        db_artist_ids = self.database.get_all_artist_ids_for_server(self.server_type) if check_artists else set()
+        db_album_ids = self.database.get_all_album_ids_for_server(self.server_type) if check_albums else set()
+
+        # Compute removal sets (only for types we have valid server data for)
+        removed_artist_ids = (db_artist_ids - server_artist_ids) if check_artists else set()
+        removed_album_ids = (db_album_ids - server_album_ids) if check_albums else set()
+
+        # Filter out albums that will already be cascade-deleted with their artist
+        if removed_artist_ids and removed_album_ids:
+            try:
+                with self.database._get_connection() as conn:
+                    cursor = conn.cursor()
+                    artist_list = list(removed_artist_ids)
+                    cascade_album_ids = set()
+                    batch_size = 500
+                    for i in range(0, len(artist_list), batch_size):
+                        batch = artist_list[i:i + batch_size]
+                        placeholders = ','.join('?' * len(batch))
+                        cursor.execute(
+                            f"SELECT id FROM albums WHERE artist_id IN ({placeholders}) "
+                            f"AND server_source = ?",
+                            batch + [self.server_type])
+                        cascade_album_ids.update(row[0] for row in cursor.fetchall())
+                    removed_album_ids -= cascade_album_ids
+            except Exception:
+                pass  # If this optimization fails, double-delete is harmless
+
+        if not removed_artist_ids and not removed_album_ids:
+            logger.info("🔍 Removal detection: no stale content found")
+            return {'artists_removed': 0, 'albums_removed': 0, 'tracks_removed': 0}
+
+        logger.info(f"🗑️ Removal detection: found {len(removed_artist_ids)} removed artists, "
+                    f"{len(removed_album_ids)} removed albums")
+
+        self._emit_signal('phase_changed',
+                         f"Removing {len(removed_artist_ids)} artists, "
+                         f"{len(removed_album_ids)} albums no longer on server...")
+
+        results = self.database.delete_removed_content(
+            removed_artist_ids, removed_album_ids, self.server_type)
+
+        self._removal_results = results
+        return results
+
     def _get_recent_albums_for_server(self) -> List:
         """Get recently added albums using server-specific methods"""
         try:

core/jellyfin_client.py

@@ -740,6 +740,65 @@ class JellyfinClient:
             logger.error(f"Error getting artists from Jellyfin: {e}")
             return []
     
+    def get_all_artist_ids(self) -> set:
+        """Get all artist IDs from Jellyfin (lightweight, for removal detection).
+        Does NOT trigger _populate_aggressive_cache()."""
+        if not self.ensure_connection() or not self.music_library_id:
+            return set()
+        try:
+            params = {
+                'ParentId': self.music_library_id,
+                'Recursive': True,
+                'Fields': '',
+                'EnableTotalRecordCount': False
+            }
+            response = self._make_request('/Artists/AlbumArtists', params)
+            if not response:
+                return set()
+            ids = {item['Id'] for item in response.get('Items', []) if 'Id' in item}
+            logger.info(f"Retrieved {len(ids)} artist IDs from Jellyfin (lightweight)")
+            return ids
+        except Exception as e:
+            logger.error(f"Error getting artist IDs from Jellyfin: {e}")
+            return set()
+
+    def get_all_album_ids(self) -> set:
+        """Get all album IDs from Jellyfin (lightweight, paginated, for removal detection).
+        Does NOT trigger _populate_aggressive_cache()."""
+        if not self.ensure_connection() or not self.music_library_id:
+            return set()
+        try:
+            all_ids = set()
+            start_index = 0
+            limit = 10000
+            while True:
+                params = {
+                    'ParentId': self.music_library_id,
+                    'IncludeItemTypes': 'MusicAlbum',
+                    'Recursive': True,
+                    'Fields': '',
+                    'SortBy': 'SortName',
+                    'SortOrder': 'Ascending',
+                    'StartIndex': start_index,
+                    'Limit': limit,
+                    'EnableTotalRecordCount': False
+                }
+                response = self._make_request(f'/Users/{self.user_id}/Items', params)
+                if not response:
+                    break
+                items = response.get('Items', [])
+                if not items:
+                    break
+                all_ids.update(item['Id'] for item in items if 'Id' in item)
+                if len(items) < limit:
+                    break
+                start_index += limit
+            logger.info(f"Retrieved {len(all_ids)} album IDs from Jellyfin (lightweight)")
+            return all_ids
+        except Exception as e:
+            logger.error(f"Error getting album IDs from Jellyfin: {e}")
+            return set()
+
     def get_albums_for_artist(self, artist_id: str) -> List[JellyfinAlbum]:
         """Get all albums for a specific artist"""
         # Use cache if available

core/navidrome_client.py

@@ -390,6 +390,64 @@ class NavidromeClient:
             logger.error(f"Error getting artists from Navidrome: {e}")
             return []
 
+    def get_all_artist_ids(self) -> set:
+        """Get all artist IDs from Navidrome (lightweight, for removal detection)."""
+        if not self.ensure_connection():
+            return set()
+        try:
+            params = {}
+            if self.music_folder_id:
+                params['musicFolderId'] = self.music_folder_id
+            response = self._make_request('getArtists', params if params else None)
+            if not response:
+                return set()
+            ids = set()
+            for index in response.get('artists', {}).get('index', []):
+                for artist_data in index.get('artist', []):
+                    aid = artist_data.get('id')
+                    if aid:
+                        ids.add(str(aid))
+            logger.info(f"Retrieved {len(ids)} artist IDs from Navidrome (lightweight)")
+            return ids
+        except Exception as e:
+            logger.error(f"Error getting artist IDs from Navidrome: {e}")
+            return set()
+
+    def get_all_album_ids(self) -> set:
+        """Get all album IDs from Navidrome (lightweight, paginated, for removal detection)."""
+        if not self.ensure_connection():
+            return set()
+        try:
+            all_ids = set()
+            page_size = 500
+            offset = 0
+            while True:
+                params = {
+                    'type': 'alphabeticalByName',
+                    'size': page_size,
+                    'offset': offset
+                }
+                if self.music_folder_id:
+                    params['musicFolderId'] = self.music_folder_id
+                response = self._make_request('getAlbumList2', params)
+                if not response:
+                    break
+                album_list = response.get('albumList2', {}).get('album', [])
+                if not album_list:
+                    break
+                for album_data in album_list:
+                    aid = album_data.get('id')
+                    if aid:
+                        all_ids.add(str(aid))
+                if len(album_list) < page_size:
+                    break
+                offset += page_size
+            logger.info(f"Retrieved {len(all_ids)} album IDs from Navidrome (lightweight)")
+            return all_ids
+        except Exception as e:
+            logger.error(f"Error getting album IDs from Navidrome: {e}")
+            return set()
+
     def get_albums_for_artist(self, artist_id: str) -> List[NavidromeAlbum]:
         """Get all albums for a specific artist"""
         # Check cache first

core/plex_client.py

@@ -617,7 +617,7 @@ class PlexClient:
         if not self.ensure_connection() or not self.music_library:
             logger.error("Not connected to Plex server or no music library")
             return []
-        
+
         try:
             artists = self.music_library.searchArtists()
             logger.info(f"Found {len(artists)} artists in Plex library")
@@ -625,7 +625,33 @@ class PlexClient:
         except Exception as e:
             logger.error(f"Error getting all artists: {e}")
             return []
-    
+
+    def get_all_artist_ids(self) -> set:
+        """Get all artist IDs from Plex library (lightweight, for removal detection)."""
+        if not self.ensure_connection() or not self.music_library:
+            return set()
+        try:
+            artists = self.music_library.searchArtists()
+            ids = {str(a.ratingKey) for a in artists}
+            logger.info(f"Retrieved {len(ids)} artist IDs from Plex")
+            return ids
+        except Exception as e:
+            logger.error(f"Error getting artist IDs from Plex: {e}")
+            return set()
+
+    def get_all_album_ids(self) -> set:
+        """Get all album IDs from Plex library (lightweight, for removal detection)."""
+        if not self.ensure_connection() or not self.music_library:
+            return set()
+        try:
+            albums = self.music_library.albums()
+            ids = {str(a.ratingKey) for a in albums}
+            logger.info(f"Retrieved {len(ids)} album IDs from Plex")
+            return ids
+        except Exception as e:
+            logger.error(f"Error getting album IDs from Plex: {e}")
+            return set()
+
     def update_artist_genres(self, artist: PlexArtist, genres: List[str]):
         """Update artist genres"""
         try:

database/music_database.py

@@ -1615,6 +1615,115 @@ class MusicDatabase:
             logger.error(f"Error merging duplicate artists: {e}")
             return {'artists_merged': 0, 'albums_migrated': 0}
 
+    # --- Removal detection helpers ---
+
+    def get_all_artist_ids_for_server(self, server_source: str) -> set:
+        """Get all artist IDs stored in the database for a specific server."""
+        try:
+            with self._get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT id FROM artists WHERE server_source = ?", (server_source,))
+                return {row[0] for row in cursor.fetchall()}
+        except Exception as e:
+            logger.error(f"Error getting artist IDs for {server_source}: {e}")
+            return set()
+
+    def get_all_album_ids_for_server(self, server_source: str) -> set:
+        """Get all album IDs stored in the database for a specific server."""
+        try:
+            with self._get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT id FROM albums WHERE server_source = ?", (server_source,))
+                return {row[0] for row in cursor.fetchall()}
+        except Exception as e:
+            logger.error(f"Error getting album IDs for {server_source}: {e}")
+            return set()
+
+    def delete_removed_content(self, removed_artist_ids: set, removed_album_ids: set,
+                               server_source: str):
+        """Delete artists and albums that were removed from the media server.
+        Manually cascades deletes (tracks -> albums -> artists) to match existing patterns."""
+        try:
+            with self._get_connection() as conn:
+                cursor = conn.cursor()
+                batch_size = 500
+
+                artists_removed = 0
+                albums_removed = 0
+                tracks_removed = 0
+
+                # Remove artists and their children
+                if removed_artist_ids:
+                    artist_list = list(removed_artist_ids)
+                    for i in range(0, len(artist_list), batch_size):
+                        batch = artist_list[i:i + batch_size]
+                        placeholders = ','.join('?' * len(batch))
+                        params = batch + [server_source]
+
+                        # Delete tracks belonging to these artists
+                        cursor.execute(
+                            f"SELECT COUNT(*) FROM tracks WHERE artist_id IN ({placeholders}) AND server_source = ?",
+                            params)
+                        tracks_removed += cursor.fetchone()[0]
+                        cursor.execute(
+                            f"DELETE FROM tracks WHERE artist_id IN ({placeholders}) AND server_source = ?",
+                            params)
+
+                        # Delete albums belonging to these artists
+                        cursor.execute(
+                            f"SELECT COUNT(*) FROM albums WHERE artist_id IN ({placeholders}) AND server_source = ?",
+                            params)
+                        albums_removed += cursor.fetchone()[0]
+                        cursor.execute(
+                            f"DELETE FROM albums WHERE artist_id IN ({placeholders}) AND server_source = ?",
+                            params)
+
+                        # Delete the artists themselves
+                        cursor.execute(
+                            f"DELETE FROM artists WHERE id IN ({placeholders}) AND server_source = ?",
+                            params)
+                        artists_removed += cursor.rowcount
+
+                # Remove albums (not already handled by artist cascade above)
+                if removed_album_ids:
+                    album_list = list(removed_album_ids)
+                    for i in range(0, len(album_list), batch_size):
+                        batch = album_list[i:i + batch_size]
+                        placeholders = ','.join('?' * len(batch))
+                        params = batch + [server_source]
+
+                        # Delete tracks belonging to these albums
+                        cursor.execute(
+                            f"SELECT COUNT(*) FROM tracks WHERE album_id IN ({placeholders}) AND server_source = ?",
+                            params)
+                        tracks_removed += cursor.fetchone()[0]
+                        cursor.execute(
+                            f"DELETE FROM tracks WHERE album_id IN ({placeholders}) AND server_source = ?",
+                            params)
+
+                        # Delete the albums themselves
+                        cursor.execute(
+                            f"DELETE FROM albums WHERE id IN ({placeholders}) AND server_source = ?",
+                            params)
+                        albums_removed += cursor.rowcount
+
+                conn.commit()
+
+                if artists_removed > 0 or albums_removed > 0:
+                    logger.info(f"Removal cleanup for {server_source}: "
+                                f"{artists_removed} artists, {albums_removed} albums, "
+                                f"{tracks_removed} tracks removed")
+
+                return {
+                    'artists_removed': artists_removed,
+                    'albums_removed': albums_removed,
+                    'tracks_removed': tracks_removed
+                }
+
+        except Exception as e:
+            logger.error(f"Error deleting removed content for {server_source}: {e}")
+            return {'artists_removed': 0, 'albums_removed': 0, 'tracks_removed': 0}
+
     # Artist operations
     def insert_or_update_artist(self, plex_artist) -> bool:
         """Insert or update artist from Plex artist object - DEPRECATED: Use insert_or_update_media_artist instead"""

web_server.py

@@ -247,7 +247,10 @@ db_update_state = {
     "current_item": "",
     "processed": 0,
     "total": 0,
-    "error_message": ""
+    "error_message": "",
+    "removed_artists": 0,
+    "removed_albums": 0,
+    "removed_tracks": 0
 }
 
 # Quality Scanner state
@@ -12626,12 +12629,30 @@ def _db_update_phase_callback(phase):
         db_update_state["phase"] = phase
 
 def _db_update_finished_callback(total_artists, total_albums, total_tracks, successful, failed):
+    # Check for removal results from the worker
+    removed_artists = 0
+    removed_albums = 0
+    removed_tracks = 0
+    if db_update_worker:
+        removed_artists = getattr(db_update_worker, 'removed_artists', 0)
+        removed_albums = getattr(db_update_worker, 'removed_albums', 0)
+        removed_tracks = getattr(db_update_worker, 'removed_tracks', 0)
+
+    removal_msg = ""
+    if removed_artists > 0 or removed_albums > 0:
+        removal_msg = f" | Removed: {removed_artists} artists, {removed_albums} albums"
+
     with db_update_lock:
         db_update_state["status"] = "finished"
-        db_update_state["phase"] = f"Completed: {successful} successful, {failed} failed."
-    
+        db_update_state["phase"] = f"Completed: {successful} successful, {failed} failed{removal_msg}."
+        db_update_state["removed_artists"] = removed_artists
+        db_update_state["removed_albums"] = removed_albums
+        db_update_state["removed_tracks"] = removed_tracks
+
     # Add activity for database update completion
     summary = f"{total_tracks} tracks, {total_albums} albums, {total_artists} artists processed"
+    if removed_artists > 0 or removed_albums > 0:
+        summary += f" | {removed_artists} artists, {removed_albums} albums removed"
     add_activity_item("✅", "Database Update Complete", summary, "Now")
     
     # WISHLIST CLEANUP: Automatically clean up wishlist after database update