From d430dd8b71038d5ac89c0dd23bc8a2da08517455 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Mon, 22 Jun 2026 20:35:20 -0700 Subject: [PATCH] #903: real source wiring for the export MBID waterfall Phase 4. core/exports/export_sources.py supplies the real I/O behind each waterfall source and assembles resolve_fn: cache -> DB (tracks.musicbrainz_recording_id via text match) -> file tag (UFID/musicbrainz_trackid) -> live MusicBrainz match_recording. Every source is fail-safe (any error -> None -> fall through, export never breaks). A fresh non-cache hit is written back to the persistent cache so the same song is free next export. Sources are injectable; build_resolve_fn wiring (cache short-circuit + write-back) is unit-tested. 4 tests. --- core/exports/export_sources.py | 184 +++++++++++++++++++++++++++ tests/exports/test_export_sources.py | 59 +++++++++ 2 files changed, 243 insertions(+) create mode 100644 core/exports/export_sources.py create mode 100644 tests/exports/test_export_sources.py diff --git a/core/exports/export_sources.py b/core/exports/export_sources.py new file mode 100644 index 00000000..48db313b --- /dev/null +++ b/core/exports/export_sources.py @@ -0,0 +1,184 @@ +"""Wire the real cheapest-first sources for the export MBID waterfall (#903). + +``mbid_resolver`` is the pure waterfall; this module supplies the real I/O behind each +source and assembles the ``resolve_fn`` the export job uses: + +1. **cache** — ``recording_mbid_cache`` (persistent (artist,title)->mbid). +2. **DB** — a text-matched library track's ``tracks.musicbrainz_recording_id``. +3. **file** — ``MUSICBRAINZ_RECORDING_ID`` tag of that track's file (when the DB row had + no recording id but the file was tagged on import). +4. **MusicBrainz** — live ``match_recording(track, artist)`` (rate-limited tail). + +Every source is wrapped so any failure (missing table, unreadable file, MB timeout) returns +None — the waterfall just falls through, the export never breaks. ``build_resolve_fn`` also +writes a fresh non-cache hit back to the cache so the next export of the same song is free. +""" + +from __future__ import annotations + +import threading +from typing import Callable, Optional, Tuple + +from utils.logging_config import get_logger + +from core.exports.mbid_resolver import ( + SRC_CACHE, + SRC_DB, + SRC_FILE, + SRC_MUSICBRAINZ, + normalize_key, + resolve_recording_mbid, +) + +logger = get_logger("exports.export_sources") + + +def _db_match(artist: str, title: str) -> Tuple[Optional[str], Optional[str]]: + """Text-match a library track by (artist, title); return (recording_mbid, file_path). + Either may be None. Fail-safe — any DB error returns (None, None).""" + if not title: + return (None, None) + try: + from database.music_database import get_database + db = get_database() + conn = db._get_connection() + try: + cur = conn.cursor() + cur.execute( + "SELECT t.musicbrainz_recording_id, t.file_path " + "FROM tracks t JOIN artists a ON t.artist_id = a.id " + "WHERE LOWER(t.title) = LOWER(?) AND LOWER(a.name) = LOWER(?) " + "LIMIT 1", + (title, artist), + ) + row = cur.fetchone() + if not row: + return (None, None) + mbid = row[0] if not hasattr(row, "keys") else row["musicbrainz_recording_id"] + fpath = row[1] if not hasattr(row, "keys") else row["file_path"] + return ((mbid or None), (fpath or None)) + finally: + try: + conn.close() + except Exception: # noqa: S110 + pass + except Exception as exc: + logger.debug(f"export db_match failed for '{artist} - {title}': {exc}") + return (None, None) + + +def db_recording_mbid(artist: str, title: str) -> Optional[str]: + """Recording MBID stored on a matched library track (``musicbrainz_recording_id``).""" + return _db_match(artist, title)[0] + + +def file_recording_mbid(artist: str, title: str) -> Optional[str]: + """Recording MBID read from the matched track's file tag (set on import post-processing).""" + _mbid, fpath = _db_match(artist, title) + if not fpath: + return None + try: + from mutagen import File as MutagenFile + audio = MutagenFile(fpath) + if audio is None or not getattr(audio, "tags", None): + return None + tags = audio.tags + # ID3 UFID (MusicBrainz), Vorbis/MP4 musicbrainz_trackid, etc. + for key in ("UFID:http://musicbrainz.org", "musicbrainz_trackid", + "MUSICBRAINZ_TRACKID", "----:com.apple.iTunes:MusicBrainz Track Id"): + try: + val = tags.get(key) + except Exception: + val = None + if not val: + continue + if hasattr(val, "data"): # ID3 UFID frame + val = val.data.decode("utf-8", "ignore") + if isinstance(val, (list, tuple)): + val = val[0] if val else "" + if isinstance(val, bytes): + val = val.decode("utf-8", "ignore") + val = str(val).strip() + if val: + return val + except Exception as exc: + logger.debug(f"export file_recording_mbid failed for {fpath}: {exc}") + return None + + +_mb_service = None +_mb_service_lock = threading.Lock() + + +def _get_mb_service(): + """Shared MusicBrainzService (client + cache + DB), created lazily so importing this + module never triggers a DB/network connection on paths that don't export.""" + global _mb_service + if _mb_service is None: + with _mb_service_lock: + if _mb_service is None: + from core.musicbrainz_service import MusicBrainzService + from database.music_database import get_database + _mb_service = MusicBrainzService(get_database()) + return _mb_service + + +def musicbrainz_recording_mbid(artist: str, title: str) -> Optional[str]: + """Live MusicBrainz ``match_recording`` — the rate-limited tail.""" + if not title: + return None + try: + svc = _get_mb_service() + if not svc: + return None + result = svc.match_recording(title, artist) + if result and result.get("mbid"): + return result["mbid"] + except Exception as exc: + logger.debug(f"export musicbrainz_recording_mbid failed for '{artist} - {title}': {exc}") + return None + + +def build_resolve_fn( + *, + db_fn: Callable[[str, str], Optional[str]] = db_recording_mbid, + file_fn: Callable[[str, str], Optional[str]] = file_recording_mbid, + mb_fn: Callable[[str, str], Optional[str]] = musicbrainz_recording_mbid, + cache_lookup: Optional[Callable[[str], Optional[str]]] = None, + cache_record: Optional[Callable[[str, str], bool]] = None, +) -> Callable[[str, str], Tuple[Optional[str], Optional[str]]]: + """Assemble the export ``resolve_fn(artist, title) -> (mbid, source_label)``. + + Runs cache -> DB -> file -> MusicBrainz, and writes a fresh (non-cache) hit back to the + persistent cache. All sources are injectable so the wiring is unit-testable; defaults + use the real cache module. + """ + if cache_lookup is None or cache_record is None: + from core.exports import recording_mbid_cache as _cache + cache_lookup = cache_lookup or _cache.lookup + cache_record = cache_record or _cache.record + + def resolve_fn(artist: str, title: str) -> Tuple[Optional[str], Optional[str]]: + sources = [ + (SRC_CACHE, lambda a, t: cache_lookup(normalize_key(a, t))), + (SRC_DB, db_fn), + (SRC_FILE, file_fn), + (SRC_MUSICBRAINZ, mb_fn), + ] + mbid, label = resolve_recording_mbid(artist, title, sources) + if mbid and label and label != SRC_CACHE: + try: + cache_record(normalize_key(artist, title), mbid) + except Exception: # noqa: S110 — cache write is best-effort + pass + return (mbid, label) + + return resolve_fn + + +__all__ = [ + "build_resolve_fn", + "db_recording_mbid", + "file_recording_mbid", + "musicbrainz_recording_mbid", +] diff --git a/tests/exports/test_export_sources.py b/tests/exports/test_export_sources.py new file mode 100644 index 00000000..bb019ea7 --- /dev/null +++ b/tests/exports/test_export_sources.py @@ -0,0 +1,59 @@ +"""Export source wiring (#903): waterfall order + cache write-back. + +build_resolve_fn assembles cache -> DB -> file -> MusicBrainz and writes a fresh +(non-cache) hit back to the cache. Pins: a cache hit short-circuits everything and is +NOT re-written; a DB/MB hit IS written back; misses fall through; the resolving label is +returned. +""" + +from __future__ import annotations + +from core.exports.export_sources import build_resolve_fn +from core.exports.mbid_resolver import SRC_CACHE, SRC_DB, SRC_MUSICBRAINZ + +MBID = "e8f9b188-f819-4e43-ab0f-4bd26ce9ff56" + + +def _wire(db=None, file=None, mb=None, cache=None): + recorded = {} + store = dict(cache or {}) + fn = build_resolve_fn( + db_fn=lambda a, t: (db or {}).get((a, t)), + file_fn=lambda a, t: (file or {}).get((a, t)), + mb_fn=lambda a, t: (mb or {}).get((a, t)), + cache_lookup=lambda k: store.get(k), + cache_record=lambda k, m: recorded.__setitem__(k, m) or True, + ) + return fn, recorded + + +def test_cache_hit_short_circuits_and_is_not_rewritten(): + from core.exports.mbid_resolver import normalize_key + fn, recorded = _wire( + cache={normalize_key("A", "T"): MBID}, + db={("A", "T"): "should-not-reach"}, + ) + mbid, label = fn("A", "T") + assert (mbid, label) == (MBID, SRC_CACHE) + assert recorded == {} # cache hit -> no write-back + + +def test_db_hit_is_written_back_to_cache(): + from core.exports.mbid_resolver import normalize_key + fn, recorded = _wire(db={("A", "T"): MBID}) + mbid, label = fn("A", "T") + assert (mbid, label) == (MBID, SRC_DB) + assert recorded == {normalize_key("A", "T"): MBID} # fresh hit cached for next time + + +def test_falls_through_to_musicbrainz_and_caches(): + fn, recorded = _wire(db={}, file={}, mb={("A", "T"): MBID}) + mbid, label = fn("A", "T") + assert (mbid, label) == (MBID, SRC_MUSICBRAINZ) + assert list(recorded.values()) == [MBID] + + +def test_all_miss_returns_none_and_no_write(): + fn, recorded = _wire() + assert fn("A", "T") == (None, None) + assert recorded == {}