From 8319c6679fbddaf00ca415f0ae630c05950a0b91 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Sat, 25 Apr 2026 13:45:46 +0300 Subject: [PATCH] Move new metadata helpers into a package - Keep existing metadata_cache and metadata_service at the top level for now - Move the new branch-local metadata helpers under core/metadata - Share MusicBrainz release cache state from core.metadata.source and update import sites --- core/imports/pipeline.py | 6 +- core/metadata/__init__.py | 2 + .../artwork.py} | 0 .../enrichment.py} | 4 +- .../lyrics.py} | 0 .../source.py} | 32 ++++++----- tests/test_metadata_enrichment.py | 6 +- web_server.py | 55 ++++++++----------- 8 files changed, 51 insertions(+), 54 deletions(-) create mode 100644 core/metadata/__init__.py rename core/{metadata_artwork.py => metadata/artwork.py} (100%) rename core/{metadata_enrichment.py => metadata/enrichment.py} (98%) rename core/{metadata_lyrics.py => metadata/lyrics.py} (100%) rename core/{metadata_source.py => metadata/source.py} (97%) diff --git a/core/imports/pipeline.py b/core/imports/pipeline.py index 68429c85..df18b81c 100644 --- a/core/imports/pipeline.py +++ b/core/imports/pipeline.py @@ -51,16 +51,16 @@ from core.runtime_state import ( processed_download_ids, tasks_lock, ) -from core.metadata_artwork import download_cover_art +from core.metadata.artwork import download_cover_art from core.metadata_common import wipe_source_tags -from core.metadata_enrichment import enhance_file_metadata +from core.metadata.enrichment import enhance_file_metadata from core.imports.paths import ( build_final_path_for_track, build_simple_download_destination, docker_resolve_path, ) from core.imports.album_naming import resolve_album_group -from core.metadata_lyrics import generate_lrc_file +from core.metadata.lyrics import generate_lrc_file from database.music_database import get_database from utils.logging_config import get_logger diff --git a/core/metadata/__init__.py b/core/metadata/__init__.py new file mode 100644 index 00000000..1961bd7b --- /dev/null +++ b/core/metadata/__init__.py @@ -0,0 +1,2 @@ +"""Metadata helper package.""" + diff --git a/core/metadata_artwork.py b/core/metadata/artwork.py similarity index 100% rename from core/metadata_artwork.py rename to core/metadata/artwork.py diff --git a/core/metadata_enrichment.py b/core/metadata/enrichment.py similarity index 98% rename from core/metadata_enrichment.py rename to core/metadata/enrichment.py index fe3fd906..0e5d5ec9 100644 --- a/core/metadata_enrichment.py +++ b/core/metadata/enrichment.py @@ -4,7 +4,7 @@ from __future__ import annotations import os -from core.metadata_artwork import embed_album_art_metadata +from core.metadata.artwork import embed_album_art_metadata from core.metadata_common import ( get_config_manager, get_file_lock, @@ -15,7 +15,7 @@ from core.metadata_common import ( strip_all_non_audio_tags, verify_metadata_written, ) -from core.metadata_source import embed_source_ids, extract_source_metadata +from core.metadata.source import embed_source_ids, extract_source_metadata __all__ = [ diff --git a/core/metadata_lyrics.py b/core/metadata/lyrics.py similarity index 100% rename from core/metadata_lyrics.py rename to core/metadata/lyrics.py diff --git a/core/metadata_source.py b/core/metadata/source.py similarity index 97% rename from core/metadata_source.py rename to core/metadata/source.py index 7f4411e9..483150ad 100644 --- a/core/metadata_source.py +++ b/core/metadata/source.py @@ -31,13 +31,17 @@ from core.metadata_common import ( __all__ = [ "extract_source_metadata", "embed_source_ids", + "normalize_album_cache_key", + "mb_release_cache", + "mb_release_cache_lock", + "mb_release_detail_cache", + "mb_release_detail_cache_lock", ] - -_MB_RELEASE_CACHE: Dict[tuple, str] = {} -_MB_RELEASE_CACHE_LOCK = threading.RLock() -_MB_RELEASE_DETAIL_CACHE: Dict[str, Dict[str, Any]] = {} -_MB_RELEASE_DETAIL_CACHE_LOCK = threading.RLock() +mb_release_cache: Dict[tuple, str] = {} +mb_release_cache_lock = threading.RLock() +mb_release_detail_cache: Dict[str, Dict[str, Any]] = {} +mb_release_detail_cache_lock = threading.RLock() _EDITION_PAREN_RE = re.compile( r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' @@ -365,10 +369,10 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), artist_key) rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) - with _MB_RELEASE_CACHE_LOCK: - cached = _MB_RELEASE_CACHE.get(rc_key_norm) + with mb_release_cache_lock: + cached = mb_release_cache.get(rc_key_norm) if cached is None: - cached = _MB_RELEASE_CACHE.get(rc_key_exact) + cached = mb_release_cache.get(rc_key_exact) if cached is not None: pp["release_mbid"] = cached else: @@ -377,21 +381,21 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" except Exception: pp["release_mbid"] = "" - _MB_RELEASE_CACHE[rc_key_norm] = pp["release_mbid"] - _MB_RELEASE_CACHE[rc_key_exact] = pp["release_mbid"] + mb_release_cache[rc_key_norm] = pp["release_mbid"] + mb_release_cache[rc_key_exact] = pp["release_mbid"] if pp["release_mbid"]: pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] if pp["release_mbid"]: - with _MB_RELEASE_DETAIL_CACHE_LOCK: - release_detail = _MB_RELEASE_DETAIL_CACHE.get(pp["release_mbid"]) + with mb_release_detail_cache_lock: + release_detail = mb_release_detail_cache.get(pp["release_mbid"]) if release_detail is None: release_detail = mb_service.mb_client.get_release( pp["release_mbid"], includes=["release-groups", "labels", "media", "artist-credits", "recordings"], ) or {} - with _MB_RELEASE_DETAIL_CACHE_LOCK: - _MB_RELEASE_DETAIL_CACHE[pp["release_mbid"]] = release_detail + with mb_release_detail_cache_lock: + mb_release_detail_cache[pp["release_mbid"]] = release_detail if release_detail: rg = release_detail.get("release-group", {}) if rg.get("id"): diff --git a/tests/test_metadata_enrichment.py b/tests/test_metadata_enrichment.py index 577bda98..35ffcd4b 100644 --- a/tests/test_metadata_enrichment.py +++ b/tests/test_metadata_enrichment.py @@ -1,8 +1,8 @@ import types -from core import metadata_enrichment as me -from core import metadata_artwork as ma -from core import metadata_source as ms +from core.metadata import enrichment as me +from core.metadata import artwork as ma +from core.metadata import source as ms class _Config: diff --git a/web_server.py b/web_server.py index daa7c687..bf7c7d4a 100644 --- a/web_server.py +++ b/web_server.py @@ -133,6 +133,13 @@ from core.imports.staging import ( ) from core.imports.paths import build_final_path_for_track as _build_final_path_for_track from core.metadata_common import get_file_lock +from core.metadata.source import ( + mb_release_cache, + mb_release_cache_lock, + mb_release_detail_cache, + mb_release_detail_cache_lock, + normalize_album_cache_key, +) from core.runtime_state import ( activity_feed, activity_feed_lock, @@ -146,8 +153,7 @@ from core.runtime_state import ( set_activity_toast_emitter, tasks_lock, ) -from core import metadata_enrichment -from core.metadata_source import normalize_album_cache_key +from core.metadata import enrichment as metadata_enrichment from database.music_database import get_database, MusicDatabase from services.sync_service import PlaylistSyncService @@ -1010,11 +1016,6 @@ session_stats_lock = threading.Lock() batch_locks = {} _orphaned_download_keys = set() -_mb_release_cache = {} -_mb_release_cache_lock = threading.Lock() -_mb_release_detail_cache = {} -_mb_release_detail_cache_lock = threading.Lock() - _enrichment_activity_log = {} _idle_since = {} _IDLE_GRACE_SECONDS = 5 @@ -5423,16 +5424,6 @@ album_artists = {} # album_key -> artist_name album_editions = {} # album_key -> "standard" or "deluxe" album_name_cache = {} # album_key -> cached_final_name -# Thread-safe cache for MusicBrainz release MBID lookups. -# Prevents concurrent post-processing threads from getting different -# release IDs for the same album, which causes players like Navidrome -# to split one album into multiple entries. -_mb_release_cache = {} # (album_lower, artist_lower) -> mbid_string_or_empty -_mb_release_cache_lock = threading.Lock() -_mb_release_detail_cache = {} # mbid -> release detail dict from get_release() -_mb_release_detail_cache_lock = threading.Lock() - - # Regexes to strip edition suffixes for cache key normalization. # Prevents Navidrome splitting albums when tracks get different MusicBrainz release IDs. import re as _re @@ -17817,11 +17808,11 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar if _pf_release and _pf_release.get('id'): _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() - with _mb_release_cache_lock: - _mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid - _mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[_pf_mbid] = _pf_release + with mb_release_cache_lock: + mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid + mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid + with mb_release_detail_cache_lock: + mb_release_detail_cache[_pf_mbid] = _pf_release logger.info(f"[Preflight] Pre-cached MB release for '{spotify_album['name']}': " f"'{_pf_release.get('title', '')}' ({_pf_mbid[:8]}...)") except Exception as pf_err: @@ -17958,11 +17949,11 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album): if _pf_release and _pf_release.get('id'): _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() - with _mb_release_cache_lock: - _mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid - _mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[_pf_mbid] = _pf_release + with mb_release_cache_lock: + mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid + mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid + with mb_release_detail_cache_lock: + mb_release_detail_cache[_pf_mbid] = _pf_release logger.info(f"[Preflight] Pre-cached MB release for '{spotify_album['name']}': " f"'{_pf_release.get('title', '')}' ({_pf_mbid[:8]}...)") except Exception as pf_err: @@ -25148,12 +25139,12 @@ def _run_full_missing_tracks_process(batch_id, playlist_id, tracks_json): _artist_key = artist_name_pf.lower().strip() _rc_key_norm = (normalize_album_cache_key(album_name_pf), _artist_key) _rc_key_exact = (album_name_pf.lower().strip(), _artist_key) - with _mb_release_cache_lock: - _mb_release_cache[_rc_key_norm] = release_mbid - _mb_release_cache[_rc_key_exact] = release_mbid + with mb_release_cache_lock: + mb_release_cache[_rc_key_norm] = release_mbid + mb_release_cache[_rc_key_exact] = release_mbid # Also cache the full release detail for tag extraction - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[release_mbid] = release + with mb_release_detail_cache_lock: + mb_release_detail_cache[release_mbid] = release logger.info(f"[Preflight] Pre-cached MB release for '{album_name_pf}': " f"'{release.get('title', '')}' ({release_mbid[:8]}...)") else: