From fabec1e45531ed64ea3ba3f68f0ddd9228701293 Mon Sep 17 00:00:00 2001 From: Broque Thomas Date: Mon, 23 Feb 2026 13:15:06 -0800 Subject: [PATCH] Handle duplicate artists and ratingKey migrations Add duplicate-artist detection/merge and handle ratingKey (ID) migrations for artists and albums. Introduces MusicDatabase.merge_duplicate_artists that picks a canonical artist (most enrichment data), merges enrichment fields, migrates albums/tracks, and removes duplicates; DatabaseUpdateWorker now runs this merge during updates (even when no new content) and after orphan cleanup. insert_or_update_artist/album now detect same-name/title + server_source collisions (ratingKey changes), inserting a new record while preserving enrichment and migrating references, with safe deletion of old rows. Also deduplicate artist listing queries so results show a single canonical row per name+server_source while aggregating album/track counts across duplicates. Logging improved to report merge/migration outcomes. --- core/database_update_worker.py | 24 ++- database/music_database.py | 292 +++++++++++++++++++++++++++++++-- 2 files changed, 297 insertions(+), 19 deletions(-) diff --git a/core/database_update_worker.py b/core/database_update_worker.py index d250abfa..54d0a818 100644 --- a/core/database_update_worker.py +++ b/core/database_update_worker.py @@ -182,7 +182,16 @@ class DatabaseUpdateWorker(QThread): self._emit_signal('phase_changed', "Finding recently added content...") artists_to_process = self._get_artists_for_incremental_update() if not artists_to_process: - logger.info("No new content found - database is up to date") + logger.info("No new content found - checking for duplicate cleanup") + # Still run duplicate merge even when no new content found + if self.database: + try: + merge_results = self.database.merge_duplicate_artists() + merged = merge_results.get('artists_merged', 0) + if merged > 0: + logger.info(f"🧹 Merged {merged} duplicate artists") + except Exception as e: + logger.warning(f"Could not merge duplicate artists: {e}") self._emit_signal('finished', 0, 0, 0, 0, 0) return logger.info(f"Incremental update: Found {len(artists_to_process)} artists to process") @@ -225,14 +234,23 @@ class DatabaseUpdateWorker(QThread): cleanup_results = self.database.cleanup_orphaned_records() orphaned_artists = cleanup_results.get('orphaned_artists_removed', 0) orphaned_albums = cleanup_results.get('orphaned_albums_removed', 0) - + if orphaned_artists > 0 or orphaned_albums > 0: logger.info(f"🧹 Cleanup complete: {orphaned_artists} orphaned artists, {orphaned_albums} orphaned albums removed") else: logger.debug("🧹 Cleanup complete: No orphaned records found") - + except Exception as e: logger.warning(f"Could not cleanup orphaned records: {e}") + + # Merge any remaining duplicate artists (same name + server_source, different IDs) + try: + merge_results = self.database.merge_duplicate_artists() + merged = merge_results.get('artists_merged', 0) + if merged > 0: + logger.info(f"🧹 Merged {merged} duplicate artists") + except Exception as e: + logger.warning(f"Could not merge duplicate artists: {e}") # Emit final results self._emit_signal('finished', diff --git a/database/music_database.py b/database/music_database.py index 18961312..618b1768 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -1438,6 +1438,122 @@ class MusicDatabase: logger.error(f"Error cleaning up orphaned records: {e}") return {'orphaned_artists_removed': 0, 'orphaned_albums_removed': 0} + def merge_duplicate_artists(self) -> Dict[str, int]: + """ + Find and merge duplicate artists that share the same name + server_source. + Keeps the artist with the most enrichment data, migrates albums/tracks, + and merges enrichment columns. + """ + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + # Find duplicate artist groups (same name + server_source, different IDs) + cursor.execute(""" + SELECT name, server_source, GROUP_CONCAT(id) as ids, COUNT(*) as cnt + FROM artists + GROUP BY name, server_source + HAVING cnt > 1 + """) + duplicate_groups = cursor.fetchall() + + if not duplicate_groups: + logger.debug("🧹 No duplicate artists found") + return {'artists_merged': 0, 'albums_migrated': 0} + + total_merged = 0 + total_albums_migrated = 0 + + enrichment_cols = [ + 'musicbrainz_id', 'musicbrainz_last_attempted', 'musicbrainz_match_status', + 'spotify_artist_id', 'spotify_match_status', 'spotify_last_attempted', + 'itunes_artist_id', 'itunes_match_status', 'itunes_last_attempted', + 'audiodb_id', 'audiodb_match_status', 'audiodb_last_attempted', + 'style', 'mood', 'label', 'banner_url', + 'deezer_id', 'deezer_match_status', 'deezer_last_attempted', + ] + + for group in duplicate_groups: + artist_name = group['name'] + server_source = group['server_source'] + ids = group['ids'].split(',') + + logger.info(f"🔄 Merging duplicate artist '{artist_name}' ({server_source}): IDs {ids}") + + # Pick the keeper: the one with the most enrichment data + best_id = ids[0] + best_score = 0 + for aid in ids: + cursor.execute("SELECT * FROM artists WHERE id = ?", (aid,)) + row = cursor.fetchone() + if row: + score = 0 + for col in enrichment_cols: + try: + if row[col] is not None: + score += 1 + except (IndexError, KeyError): + continue + if score > best_score: + best_score = score + best_id = aid + + # Merge enrichment data from all duplicates into the keeper + for aid in ids: + if aid == best_id: + continue + cursor.execute("SELECT * FROM artists WHERE id = ?", (aid,)) + donor = cursor.fetchone() + if not donor: + continue + + # Fill NULL enrichment columns on keeper from this duplicate + set_parts = [] + values = [] + for col in enrichment_cols: + try: + donor_val = donor[col] + if donor_val is not None: + # Only fill if keeper's value is NULL + set_parts.append(f"{col} = COALESCE({col}, ?)") + values.append(donor_val) + except (IndexError, KeyError): + continue + + if set_parts: + values.append(best_id) + cursor.execute(f""" + UPDATE artists SET {', '.join(set_parts)} + WHERE id = ? + """, values) + + # Migrate albums and tracks from duplicate to keeper + cursor.execute("UPDATE albums SET artist_id = ? WHERE artist_id = ?", (best_id, aid)) + migrated = cursor.rowcount + total_albums_migrated += migrated + cursor.execute("UPDATE tracks SET artist_id = ? WHERE artist_id = ?", (best_id, aid)) + + # Delete the duplicate artist + cursor.execute("SELECT COUNT(*) FROM albums WHERE artist_id = ?", (aid,)) + remaining = cursor.fetchone()[0] + if remaining == 0: + cursor.execute("DELETE FROM artists WHERE id = ?", (aid,)) + total_merged += 1 + logger.info(f" Merged '{artist_name}' ID {aid} → {best_id} ({migrated} albums migrated)") + else: + logger.warning(f" Could not delete duplicate {aid}: {remaining} albums still reference it") + + conn.commit() + + if total_merged > 0: + logger.info(f"🧹 Duplicate merge complete: {total_merged} duplicates merged, {total_albums_migrated} albums migrated") + + return {'artists_merged': total_merged, 'albums_migrated': total_albums_migrated} + + except Exception as e: + logger.error(f"Error merging duplicate artists: {e}") + return {'artists_merged': 0, 'albums_migrated': 0} + # Artist operations def insert_or_update_artist(self, plex_artist) -> bool: """Insert or update artist from Plex artist object - DEPRECATED: Use insert_or_update_media_artist instead""" @@ -1491,7 +1607,7 @@ class MusicDatabase: # Check if artist exists with this ID and server source cursor.execute("SELECT id FROM artists WHERE id = ? AND server_source = ?", (artist_id, server_source)) exists = cursor.fetchone() - + if exists: # Update existing artist cursor.execute(""" @@ -1501,12 +1617,80 @@ class MusicDatabase: """, (name, thumb_url, genres_json, summary, artist_id, server_source)) logger.debug(f"Updated existing {server_source} artist: {name} (ID: {artist_id})") else: - # Insert new artist - cursor.execute(""" - INSERT INTO artists (id, name, thumb_url, genres, summary, server_source) - VALUES (?, ?, ?, ?, ?, ?) - """, (artist_id, name, thumb_url, genres_json, summary, server_source)) - logger.debug(f"Inserted new {server_source} artist: {name} (ID: {artist_id})") + # Before inserting, check if an artist with the same name already exists + # for this server source (ratingKey may have changed after a library rescan) + cursor.execute("SELECT id FROM artists WHERE name = ? AND server_source = ?", (name, server_source)) + existing_by_name = cursor.fetchone() + + if existing_by_name: + old_id = existing_by_name['id'] + # ratingKey changed — migrate old artist to new ID, preserving enrichment data + logger.info(f"🔄 Artist ratingKey migrated: '{name}' ({old_id} → {artist_id})") + + # Step 1: Insert new artist record, copying enrichment data from old + enrichment_cols = [ + 'musicbrainz_id', 'musicbrainz_last_attempted', 'musicbrainz_match_status', + 'spotify_artist_id', 'spotify_match_status', 'spotify_last_attempted', + 'itunes_artist_id', 'itunes_match_status', 'itunes_last_attempted', + 'audiodb_id', 'audiodb_match_status', 'audiodb_last_attempted', + 'style', 'mood', 'label', 'banner_url', + 'deezer_id', 'deezer_match_status', 'deezer_last_attempted', + ] + + # Read enrichment data from old artist + cursor.execute("SELECT * FROM artists WHERE id = ? AND server_source = ?", (old_id, server_source)) + old_row = cursor.fetchone() + + # Insert new artist with fresh server metadata + preserved created_at + old_created = old_row['created_at'] if old_row else None + cursor.execute(""" + INSERT INTO artists (id, name, thumb_url, genres, summary, server_source, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + """, (artist_id, name, thumb_url, genres_json, summary, server_source, old_created)) + + # Copy enrichment data from old record to new record + if old_row: + set_parts = [] + values = [] + for col in enrichment_cols: + try: + val = old_row[col] + if val is not None: + set_parts.append(f"{col} = ?") + values.append(val) + except (IndexError, KeyError): + continue # Column doesn't exist in this DB version + + if set_parts: + values.append(artist_id) + cursor.execute(f""" + UPDATE artists SET {', '.join(set_parts)} + WHERE id = ? + """, values) + + # Step 2: Migrate album and track references to new artist ID + cursor.execute("UPDATE albums SET artist_id = ? WHERE artist_id = ?", (artist_id, old_id)) + migrated_albums = cursor.rowcount + cursor.execute("UPDATE tracks SET artist_id = ? WHERE artist_id = ?", (artist_id, old_id)) + migrated_tracks = cursor.rowcount + + # Step 3: Safely delete old artist (verify no remaining references first) + cursor.execute("SELECT COUNT(*) FROM albums WHERE artist_id = ?", (old_id,)) + remaining = cursor.fetchone()[0] + if remaining == 0: + cursor.execute("DELETE FROM artists WHERE id = ? AND server_source = ?", (old_id, server_source)) + else: + logger.warning(f"Could not delete old artist {old_id}: {remaining} albums still reference it") + + if migrated_albums > 0 or migrated_tracks > 0: + logger.info(f" Migrated {migrated_albums} albums, {migrated_tracks} tracks to new ID") + else: + # Genuinely new artist — insert fresh record + cursor.execute(""" + INSERT INTO artists (id, name, thumb_url, genres, summary, server_source) + VALUES (?, ?, ?, ?, ?, ?) + """, (artist_id, name, thumb_url, genres_json, summary, server_source)) + logger.debug(f"Inserted new {server_source} artist: {name} (ID: {artist_id})") conn.commit() rows_affected = cursor.rowcount @@ -1602,12 +1786,81 @@ class MusicDatabase: WHERE id = ? """, (artist_id, title, year, thumb_url, genres_json, track_count, duration, server_source, album_id)) else: - # Insert new album - cursor.execute(""" - INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, track_count, duration, server_source) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - """, (album_id, artist_id, title, year, thumb_url, genres_json, track_count, duration, server_source)) - + # Before inserting, check if an album with the same title already exists + # under this artist (ratingKey may have changed after a library rescan) + cursor.execute( + "SELECT id FROM albums WHERE title = ? AND artist_id = ? AND server_source = ?", + (title, artist_id, server_source)) + existing_by_title = cursor.fetchone() + + if existing_by_title: + old_id = existing_by_title['id'] + # ratingKey changed — migrate old album to new ID, preserving enrichment data + logger.info(f"🔄 Album ratingKey migrated: '{title}' ({old_id} → {album_id})") + + enrichment_cols = [ + 'musicbrainz_release_id', 'musicbrainz_last_attempted', 'musicbrainz_match_status', + 'spotify_album_id', 'spotify_match_status', 'spotify_last_attempted', + 'itunes_album_id', 'itunes_match_status', 'itunes_last_attempted', + 'audiodb_id', 'audiodb_match_status', 'audiodb_last_attempted', + 'style', 'mood', 'label', 'explicit', 'record_type', + 'deezer_id', 'deezer_match_status', 'deezer_last_attempted', + ] + + # Read enrichment data from old album + cursor.execute("SELECT * FROM albums WHERE id = ?", (old_id,)) + old_row = cursor.fetchone() + + # Insert new album with fresh server metadata + preserved created_at + old_created = old_row['created_at'] if old_row else None + cursor.execute(""" + INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, + track_count, duration, server_source, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + """, (album_id, artist_id, title, year, thumb_url, genres_json, + track_count, duration, server_source, old_created)) + + # Copy enrichment data from old record to new record + if old_row: + set_parts = [] + values = [] + for col in enrichment_cols: + try: + val = old_row[col] + if val is not None: + set_parts.append(f"{col} = ?") + values.append(val) + except (IndexError, KeyError): + continue # Column doesn't exist in this DB version + + if set_parts: + values.append(album_id) + cursor.execute(f""" + UPDATE albums SET {', '.join(set_parts)} + WHERE id = ? + """, values) + + # Migrate track references to new album ID + cursor.execute("UPDATE tracks SET album_id = ? WHERE album_id = ?", (album_id, old_id)) + migrated_tracks = cursor.rowcount + + # Safely delete old album (verify no remaining references first) + cursor.execute("SELECT COUNT(*) FROM tracks WHERE album_id = ?", (old_id,)) + remaining = cursor.fetchone()[0] + if remaining == 0: + cursor.execute("DELETE FROM albums WHERE id = ?", (old_id,)) + else: + logger.warning(f"Could not delete old album {old_id}: {remaining} tracks still reference it") + + if migrated_tracks > 0: + logger.info(f" Migrated {migrated_tracks} tracks to new album ID") + else: + # Genuinely new album — insert fresh record + cursor.execute(""" + INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, track_count, duration, server_source) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, (album_id, artist_id, title, year, thumb_url, genres_json, track_count, duration, server_source)) + conn.commit() return True @@ -4403,11 +4656,13 @@ class MusicDatabase: # Empty watchlist, no artists can match where_clause += " AND 0" - # Get total count + # Deduplicate: only count canonical artist rows (one per name+server_source) count_query = f""" SELECT COUNT(*) as total_count FROM artists a WHERE {where_clause} + AND a.id = (SELECT MIN(a2.id) FROM artists a2 + WHERE a2.name = a.name AND a2.server_source = a.server_source) """ cursor.execute(count_query, params) total_count = cursor.fetchone()['total_count'] @@ -4415,6 +4670,8 @@ class MusicDatabase: # Get artists with pagination offset = (page - 1) * limit + # Deduplicate: select canonical row per name, but count albums/tracks + # across ALL same-name artist IDs (fixes duplicate artist display) artists_query = f""" SELECT a.id, @@ -4429,14 +4686,17 @@ class MusicDatabase: COUNT(DISTINCT al.id) as album_count, COUNT(DISTINCT t.id) as track_count FROM artists a - LEFT JOIN albums al ON a.id = al.artist_id + LEFT JOIN albums al ON al.artist_id IN ( + SELECT id FROM artists WHERE name = a.name AND server_source = a.server_source + ) LEFT JOIN tracks t ON al.id = t.album_id WHERE {where_clause} + AND a.id = (SELECT MIN(a2.id) FROM artists a2 + WHERE a2.name = a.name AND a2.server_source = a.server_source) GROUP BY a.id, a.name, a.thumb_url, a.genres, a.musicbrainz_id, a.deezer_id, a.audiodb_id ORDER BY a.name COLLATE NOCASE LIMIT ? OFFSET ? """ - # No need for complex query params now query_params = params + [limit, offset] cursor.execute(artists_query, query_params)