From 55c9b52aee7fb836e9435cae07649f1dfb99dd81 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 5 Jun 2026 10:21:52 -0700 Subject: [PATCH] Auto-repair duplicated source ids on startup (one-time migration) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ships the source-id cleanup to all users: a marker-gated one-time migration in MusicDatabase init clears any source id (deezer/spotify/itunes/musicbrainz/ discogs/audiodb/qobuz/tidal) shared across differently-named artists — the enrichment-corruption signature. Same-name cross-server duplicates are left untouched (DISTINCT-name check). Cleared rows re-derive correct ids on the next enrichment pass; the now name-guarded workers won't re-corrupt. Runs once (CREATE TABLE _source_id_dedupe_v1 marker), idempotent, per-column try/except so a missing column can't abort it. Test forces a re-run and asserts corruption is cleared while a legit same-name dup survives. --- database/music_database.py | 49 +++++++++++++++++++++++++++++++++ tests/test_dedupe_source_ids.py | 37 +++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/database/music_database.py b/database/music_database.py index 3e52c2dc..e0fcdbe1 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -790,6 +790,55 @@ class MusicDatabase: except Exception as e: logger.debug("Failed to purge cached tracks/albums with junk artist names: %s", e) + # One-time migration: clear source ids that enrichment wrongly + # SHARED across differently-named artists. The album/track "artist + # id correction" path (Deezer/AudioDB/Qobuz/Tidal) used to overwrite + # an artist's source id from a match without a name check, so e.g. + # everyone featured on Kendrick Lamar's curated "Black Panther" album + # got stamped with Kendrick's Deezer id. The workers are now + # name-guarded so this can't recur; clearing the bad rows lets the + # next enrichment pass re-derive each artist's correct id. + # Same-name duplicates (one artist indexed on two media servers, + # legitimately sharing an id) are left alone via the DISTINCT-name + # check, so this only touches genuine corruption. + try: + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='_source_id_dedupe_v1'") + if not cursor.fetchone(): + _dedupe_id_cols = [ + ('deezer_id', 'deezer_match_status'), + ('spotify_artist_id', 'spotify_match_status'), + ('itunes_artist_id', 'itunes_match_status'), + ('musicbrainz_id', 'musicbrainz_match_status'), + ('discogs_id', 'discogs_match_status'), + ('audiodb_id', 'audiodb_match_status'), + ('qobuz_id', 'qobuz_match_status'), + ('tidal_id', 'tidal_match_status'), + ] + total_cleared = 0 + for id_col, status_col in _dedupe_id_cols: + try: + cursor.execute(f""" + UPDATE artists + SET {id_col} = NULL, {status_col} = NULL + WHERE {id_col} IN ( + SELECT {id_col} FROM artists + WHERE {id_col} IS NOT NULL AND {id_col} != '' + GROUP BY {id_col} + HAVING COUNT(DISTINCT LOWER(TRIM(name))) > 1 + ) + """) + total_cleared += cursor.rowcount + except Exception as col_err: + logger.debug("Source-id dedupe skipped %s: %s", id_col, col_err) + cursor.execute("CREATE TABLE _source_id_dedupe_v1 (applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)") + if total_cleared > 0: + logger.info( + f"Cleared {total_cleared} duplicated source ids shared across " + f"differently-named artists — they'll re-derive on next enrichment" + ) + except Exception as e: + logger.debug("Failed to dedupe shared source ids: %s", e) + # HiFi API instances table cursor.execute(""" CREATE TABLE IF NOT EXISTS hifi_instances ( diff --git a/tests/test_dedupe_source_ids.py b/tests/test_dedupe_source_ids.py index ac98be28..59e4dc06 100644 --- a/tests/test_dedupe_source_ids.py +++ b/tests/test_dedupe_source_ids.py @@ -9,6 +9,7 @@ from __future__ import annotations import pytest +import database.music_database as mdb_mod from core.maintenance import dedupe_source_ids as dd from database.music_database import MusicDatabase @@ -129,3 +130,39 @@ def test_clean_library_is_a_noop(db): report = dd.clear_corrupt_source_ids(db, dry_run=False) assert report['cluster_count'] == 0 assert report['artist_count'] == 0 + + +# --------------------------------------------------------------------------- +# The one-time startup migration (auto-repair for users who pull the fix) +# --------------------------------------------------------------------------- + +def test_startup_migration_clears_shared_source_ids(tmp_path): + """The _source_id_dedupe_v1 migration in MusicDatabase init must clear + differently-named shared ids and leave same-name cross-server dups.""" + path = str(tmp_path / "music.db") + db = MusicDatabase(path) # first init creates the marker on an empty db + + with db._get_connection() as conn: + conn.execute("INSERT INTO artists (id,name,server_source,deezer_id,deezer_match_status) " + "VALUES ('1','Kendrick Lamar','plex','525046','matched')") + conn.execute("INSERT INTO artists (id,name,server_source,deezer_id,deezer_match_status) " + "VALUES ('2','Jorja Smith','plex','525046','matched')") + # Legit same-name dup across two servers — must survive. + conn.execute("INSERT INTO artists (id,name,server_source,spotify_artist_id) " + "VALUES ('3','Radiohead','plex','rh')") + conn.execute("INSERT INTO artists (id,name,server_source,spotify_artist_id) " + "VALUES ('4','Radiohead','jellyfin','rh')") + conn.execute("DROP TABLE _source_id_dedupe_v1") + conn.commit() + + # Force the one-time migration to run again. + mdb_mod._database_initialized_paths.clear() + MusicDatabase(path) + + with db._get_connection() as conn: + k = conn.execute("SELECT deezer_id, deezer_match_status FROM artists WHERE id='1'").fetchone() + assert tuple(k) == (None, None) + assert conn.execute("SELECT deezer_id FROM artists WHERE id='2'").fetchone()[0] is None + rh = conn.execute("SELECT spotify_artist_id FROM artists WHERE id IN ('3','4')").fetchall() + assert all(r[0] == 'rh' for r in rh) + assert conn.execute("SELECT name FROM sqlite_master WHERE name='_source_id_dedupe_v1'").fetchone()