Auto-repair duplicated source ids on startup (one-time migration)
Ships the source-id cleanup to all users: a marker-gated one-time migration in MusicDatabase init clears any source id (deezer/spotify/itunes/musicbrainz/ discogs/audiodb/qobuz/tidal) shared across differently-named artists — the enrichment-corruption signature. Same-name cross-server duplicates are left untouched (DISTINCT-name check). Cleared rows re-derive correct ids on the next enrichment pass; the now name-guarded workers won't re-corrupt. Runs once (CREATE TABLE _source_id_dedupe_v1 marker), idempotent, per-column try/except so a missing column can't abort it. Test forces a re-run and asserts corruption is cleared while a legit same-name dup survives.
This commit is contained in:
parent
0af99881bf
commit
55c9b52aee
2 changed files with 86 additions and 0 deletions
|
|
@ -790,6 +790,55 @@ class MusicDatabase:
|
|||
except Exception as e:
|
||||
logger.debug("Failed to purge cached tracks/albums with junk artist names: %s", e)
|
||||
|
||||
# One-time migration: clear source ids that enrichment wrongly
|
||||
# SHARED across differently-named artists. The album/track "artist
|
||||
# id correction" path (Deezer/AudioDB/Qobuz/Tidal) used to overwrite
|
||||
# an artist's source id from a match without a name check, so e.g.
|
||||
# everyone featured on Kendrick Lamar's curated "Black Panther" album
|
||||
# got stamped with Kendrick's Deezer id. The workers are now
|
||||
# name-guarded so this can't recur; clearing the bad rows lets the
|
||||
# next enrichment pass re-derive each artist's correct id.
|
||||
# Same-name duplicates (one artist indexed on two media servers,
|
||||
# legitimately sharing an id) are left alone via the DISTINCT-name
|
||||
# check, so this only touches genuine corruption.
|
||||
try:
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='_source_id_dedupe_v1'")
|
||||
if not cursor.fetchone():
|
||||
_dedupe_id_cols = [
|
||||
('deezer_id', 'deezer_match_status'),
|
||||
('spotify_artist_id', 'spotify_match_status'),
|
||||
('itunes_artist_id', 'itunes_match_status'),
|
||||
('musicbrainz_id', 'musicbrainz_match_status'),
|
||||
('discogs_id', 'discogs_match_status'),
|
||||
('audiodb_id', 'audiodb_match_status'),
|
||||
('qobuz_id', 'qobuz_match_status'),
|
||||
('tidal_id', 'tidal_match_status'),
|
||||
]
|
||||
total_cleared = 0
|
||||
for id_col, status_col in _dedupe_id_cols:
|
||||
try:
|
||||
cursor.execute(f"""
|
||||
UPDATE artists
|
||||
SET {id_col} = NULL, {status_col} = NULL
|
||||
WHERE {id_col} IN (
|
||||
SELECT {id_col} FROM artists
|
||||
WHERE {id_col} IS NOT NULL AND {id_col} != ''
|
||||
GROUP BY {id_col}
|
||||
HAVING COUNT(DISTINCT LOWER(TRIM(name))) > 1
|
||||
)
|
||||
""")
|
||||
total_cleared += cursor.rowcount
|
||||
except Exception as col_err:
|
||||
logger.debug("Source-id dedupe skipped %s: %s", id_col, col_err)
|
||||
cursor.execute("CREATE TABLE _source_id_dedupe_v1 (applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)")
|
||||
if total_cleared > 0:
|
||||
logger.info(
|
||||
f"Cleared {total_cleared} duplicated source ids shared across "
|
||||
f"differently-named artists — they'll re-derive on next enrichment"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("Failed to dedupe shared source ids: %s", e)
|
||||
|
||||
# HiFi API instances table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS hifi_instances (
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from __future__ import annotations
|
|||
|
||||
import pytest
|
||||
|
||||
import database.music_database as mdb_mod
|
||||
from core.maintenance import dedupe_source_ids as dd
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
|
|
@ -129,3 +130,39 @@ def test_clean_library_is_a_noop(db):
|
|||
report = dd.clear_corrupt_source_ids(db, dry_run=False)
|
||||
assert report['cluster_count'] == 0
|
||||
assert report['artist_count'] == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The one-time startup migration (auto-repair for users who pull the fix)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_startup_migration_clears_shared_source_ids(tmp_path):
|
||||
"""The _source_id_dedupe_v1 migration in MusicDatabase init must clear
|
||||
differently-named shared ids and leave same-name cross-server dups."""
|
||||
path = str(tmp_path / "music.db")
|
||||
db = MusicDatabase(path) # first init creates the marker on an empty db
|
||||
|
||||
with db._get_connection() as conn:
|
||||
conn.execute("INSERT INTO artists (id,name,server_source,deezer_id,deezer_match_status) "
|
||||
"VALUES ('1','Kendrick Lamar','plex','525046','matched')")
|
||||
conn.execute("INSERT INTO artists (id,name,server_source,deezer_id,deezer_match_status) "
|
||||
"VALUES ('2','Jorja Smith','plex','525046','matched')")
|
||||
# Legit same-name dup across two servers — must survive.
|
||||
conn.execute("INSERT INTO artists (id,name,server_source,spotify_artist_id) "
|
||||
"VALUES ('3','Radiohead','plex','rh')")
|
||||
conn.execute("INSERT INTO artists (id,name,server_source,spotify_artist_id) "
|
||||
"VALUES ('4','Radiohead','jellyfin','rh')")
|
||||
conn.execute("DROP TABLE _source_id_dedupe_v1")
|
||||
conn.commit()
|
||||
|
||||
# Force the one-time migration to run again.
|
||||
mdb_mod._database_initialized_paths.clear()
|
||||
MusicDatabase(path)
|
||||
|
||||
with db._get_connection() as conn:
|
||||
k = conn.execute("SELECT deezer_id, deezer_match_status FROM artists WHERE id='1'").fetchone()
|
||||
assert tuple(k) == (None, None)
|
||||
assert conn.execute("SELECT deezer_id FROM artists WHERE id='2'").fetchone()[0] is None
|
||||
rh = conn.execute("SELECT spotify_artist_id FROM artists WHERE id IN ('3','4')").fetchall()
|
||||
assert all(r[0] == 'rh' for r in rh)
|
||||
assert conn.execute("SELECT name FROM sqlite_master WHERE name='_source_id_dedupe_v1'").fetchone()
|
||||
|
|
|
|||
Loading…
Reference in a new issue