From c5b02c002641e8714b92e29c6394e5fbdb106e5c Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 12:11:09 -0700 Subject: [PATCH] DB: normalize legacy comma-separated genres to canonical JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit artists.genres / albums.genres stored EITHER a JSON array (new writes) OR a legacy comma-separated string (old writes), forcing every reader to try-JSON-then-split. Add a marker-gated one-time migration (_normalize_genres_to_json) that rewrites legacy rows to JSON in place, mirroring the readers' exact parse (JSON list, else comma-split/strip/ drop-empties) so genre VALUES are unchanged — only the storage format. Per-row diffed (already-canonical rows untouched, no churn) and non-fatal on error, consistent with the other migrations. Readers still tolerate both formats, so this breaks nothing; it just removes the dual-format debt. Tests: tests/test_db_genres_json_normalization.py — CSV->JSON, JSON-unchanged, whitespace/empties dropped, albums table, legacy-reader-equivalence, idempotent re-run, marker set on fresh init. --- database/music_database.py | 52 +++++++++ tests/test_db_genres_json_normalization.py | 120 +++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 tests/test_db_genres_json_normalization.py diff --git a/database/music_database.py b/database/music_database.py index ebba3893..523d7cdf 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -793,6 +793,7 @@ class MusicDatabase: logger.error(f"Personalized-playlist schema init failed: {ps_err}") self._ensure_core_media_schema_columns(cursor) + self._normalize_genres_to_json(cursor) conn.commit() logger.info("Database initialized successfully") @@ -803,6 +804,57 @@ class MusicDatabase: self._init_manual_library_match_table() + def _normalize_genres_to_json(self, cursor): + """One-time: rewrite legacy comma-separated genres to canonical JSON arrays. + + ``artists.genres`` / ``albums.genres`` historically stored EITHER a JSON + array (new writes) OR a comma-separated string (old writes), so every + reader has to try-JSON-then-split. This normalizes existing rows to JSON + in place. It mirrors the readers' exact parse (JSON list, else + comma-split/strip/drop-empties), so the genre VALUES are unchanged — only + the storage format. Marker-gated to run once, and per-row diffed so a + re-run (or a row already in JSON form) is a no-op. Non-fatal on error, + like the other migrations. + """ + import json + try: + cursor.execute("SELECT value FROM metadata WHERE key = 'genres_json_normalized' LIMIT 1") + if cursor.fetchone(): + return + + def _to_list(raw): + # Identical semantics to the genres readers elsewhere in this file. + try: + parsed = json.loads(raw) + return parsed if isinstance(parsed, list) else [str(parsed)] + except (json.JSONDecodeError, ValueError, TypeError): + return [g.strip() for g in raw.split(',') if g.strip()] + + total = 0 + for table in ('artists', 'albums'): + cursor.execute( + f"SELECT id, genres FROM {table} " + f"WHERE genres IS NOT NULL AND TRIM(genres) != ''" + ) + pending = [] + for row in cursor.fetchall(): + rid, raw = row[0], row[1] + canonical = json.dumps(_to_list(raw)) + if canonical != raw: # leave already-canonical rows untouched + pending.append((canonical, rid)) + for canonical, rid in pending: + cursor.execute(f"UPDATE {table} SET genres = ? WHERE id = ?", (canonical, rid)) + total += 1 + + cursor.execute( + "INSERT OR REPLACE INTO metadata (key, value, updated_at) " + "VALUES ('genres_json_normalized', 'true', CURRENT_TIMESTAMP)" + ) + if total: + logger.info("Normalized %d legacy genres value(s) to JSON", total) + except Exception as e: + logger.error(f"Error normalizing genres to JSON: {e}") + def _ensure_core_media_schema_columns(self, cursor): """Repair required media-library columns that older migrations may miss. diff --git a/tests/test_db_genres_json_normalization.py b/tests/test_db_genres_json_normalization.py new file mode 100644 index 00000000..078ab2a7 --- /dev/null +++ b/tests/test_db_genres_json_normalization.py @@ -0,0 +1,120 @@ +"""Tests for the one-time genres CSV->JSON normalization migration. + +artists.genres / albums.genres historically stored either a JSON array (new +writes) or a legacy comma-separated string (old writes). _normalize_genres_to_json +rewrites legacy rows to canonical JSON, mirroring the readers' exact parse so the +genre VALUES are unchanged — only the storage format. These tests drive the real +method on a temp database. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from database.music_database import MusicDatabase + + +def _fresh_db(tmp_path: Path) -> MusicDatabase: + # Init creates the schema and (harmlessly) runs the normalization once on the + # empty DB, setting the marker. Tests clear the marker + seed, then call the + # method directly so they exercise the real normalization logic. + return MusicDatabase(str(tmp_path / "library.db")) + + +def _seed_and_normalize(db: MusicDatabase, artists, albums=()): + """Insert (id, name, genres) artists and (id, artist_id, title, genres) albums + with the marker cleared, then run the real migration. Returns nothing.""" + with db._get_connection() as conn: + cur = conn.cursor() + cur.execute("DELETE FROM metadata WHERE key = 'genres_json_normalized'") + for aid, name, genres in artists: + cur.execute( + "INSERT INTO artists (id, name, genres) VALUES (?, ?, ?)", + (aid, name, genres), + ) + for alid, artist_id, title, genres in albums: + cur.execute( + "INSERT INTO albums (id, artist_id, title, genres) VALUES (?, ?, ?, ?)", + (alid, artist_id, title, genres), + ) + conn.commit() + db._normalize_genres_to_json(cur) + conn.commit() + + +def _get_genres(db: MusicDatabase, table: str, rid: str): + with db._get_connection() as conn: + row = conn.execute(f"SELECT genres FROM {table} WHERE id = ?", (rid,)).fetchone() + return row[0] + + +def test_csv_genres_normalized_to_json(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + _seed_and_normalize(db, [("a1", "Artist One", "Rock, Pop, Jazz")]) + stored = _get_genres(db, "artists", "a1") + assert json.loads(stored) == ["Rock", "Pop", "Jazz"] + + +def test_existing_json_genres_left_unchanged(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + canonical = json.dumps(["Hip-Hop", "Soul"]) + _seed_and_normalize(db, [("a1", "Artist One", canonical)]) + # Byte-for-byte identical — no needless churn on already-canonical rows. + assert _get_genres(db, "artists", "a1") == canonical + + +def test_single_genre_without_comma(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + _seed_and_normalize(db, [("a1", "Artist One", "Electronic")]) + assert json.loads(_get_genres(db, "artists", "a1")) == ["Electronic"] + + +def test_csv_whitespace_and_empties_dropped(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + _seed_and_normalize(db, [("a1", "Artist One", " Rock ,, Pop , ")]) + assert json.loads(_get_genres(db, "artists", "a1")) == ["Rock", "Pop"] + + +def test_albums_table_also_normalized(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + _seed_and_normalize( + db, + artists=[("a1", "Artist One", "Rock")], + albums=[("al1", "a1", "Album One", "Soul, Funk")], + ) + assert json.loads(_get_genres(db, "albums", "al1")) == ["Soul", "Funk"] + + +def test_values_match_legacy_reader_semantics(tmp_path: Path) -> None: + """The normalized list must equal what the legacy CSV reader would produce, + so downstream genre values are identical pre- and post-migration.""" + db = _fresh_db(tmp_path) + raw = "Rock, Pop, Hip-Hop/Rap" + _seed_and_normalize(db, [("a1", "Artist One", raw)]) + legacy = [g.strip() for g in raw.split(",") if g.strip()] + assert json.loads(_get_genres(db, "artists", "a1")) == legacy + + +def test_idempotent_rerun(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + _seed_and_normalize(db, [("a1", "Artist One", "Rock, Pop")]) + first = _get_genres(db, "artists", "a1") + # Marker is now set; a second run must be a no-op and leave the value identical. + with db._get_connection() as conn: + cur = conn.cursor() + db._normalize_genres_to_json(cur) + conn.commit() + assert _get_genres(db, "artists", "a1") == first + assert json.loads(first) == ["Rock", "Pop"] + + +def test_marker_set_after_fresh_init(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + with db._get_connection() as conn: + row = conn.execute( + "SELECT value FROM metadata WHERE key = 'genres_json_normalized'" + ).fetchone() + assert row is not None and row[0] == "true"