From 4743dfd64477a4bfa3ef1b20d16e20a78476e824 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Sun, 28 Jun 2026 23:02:08 -0700 Subject: [PATCH] playlist export: opt-in confident-search backfill for the unmatched tail (#945, increment 7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the third resolver stage for tracks the discovery cache + library can't resolve — a live search of the target service, gated behind a "Match missing tracks" toggle so the API cost is opt-in. The whole point is coverage WITHOUT the wrong-track risk, so it's a CONFIDENT match, not "search and grab": - search_service_track_id(artist, title, search_fn): searches the service, reranks via the existing relevance scorer (filter_and_rerank), and returns the top hit's id ONLY if it clears BACKFILL_MIN_SCORE (1.2 on the score_track scale). A wrong-artist hit (no 1.5x exact-artist boost, caps ~1.0) or a karaoke/cover (x0.05) can't clear the floor → None, and the track is left out rather than added wrong. search_fn injected → unit-testable without a live service. - resolve_service_track_ids gains an optional search_id_fn: cache → library → search. Tallies from_search separately. - _run_service_export builds the search fn from the service's metadata search client only when job['backfill'] is set; the endpoint reads `backfill` from the body; the modal adds the toggle and the status line shows "(N matched live)". Store-back of confident matches deferred: a mirrored-only track may have no library row to write to, so persisting needs the track→library mapping — a follow-up, not correctness. 9 new tests incl. the safety ones: wrong-artist rejected, karaoke/cover rejected, real-over-cover picked, fail-safe on search error, and the cache→library→search waterfall + toggle wiring (on/off). 28 export/orchestration tests green, 64 script-integrity green, ruff clean. --- core/exports/export_sources.py | 57 ++++++++++++++++++++-- tests/exports/test_export_sources.py | 69 ++++++++++++++++++++++++++- tests/test_service_playlist_export.py | 28 +++++++++-- web_server.py | 38 ++++++++++++--- webui/static/stats-automations.js | 14 ++++-- 5 files changed, 186 insertions(+), 20 deletions(-) diff --git a/core/exports/export_sources.py b/core/exports/export_sources.py index dbf5bbd8..90c028d6 100644 --- a/core/exports/export_sources.py +++ b/core/exports/export_sources.py @@ -162,21 +162,25 @@ def resolve_service_track_ids( service: str, *, db_fn: Optional[Callable[[str, str, str], Optional[str]]] = None, + search_id_fn: Optional[Callable[[str, str], Optional[str]]] = None, on_progress: Optional[Callable[[int, int, Dict[str, Any]], None]] = None, ) -> Dict[str, Any]: """Resolve a mirrored playlist's tracks to target-service track IDs for export. Waterfall per track: the discovery cache (``extra_data`` — free + already confidently - matched) → the library track's stored service id. A track with neither is reported - unmatched (caller skips it — never a guessed/wrong id). Returns - ``{"resolved": [{artist, title, album, service_track_id}], "stats": {...}}`` with - ``from_cache`` / ``from_library`` / ``unmatched`` tallies for the status display. + matched) → the library track's stored service id → (only when ``search_id_fn`` is + given, i.e. the opt-in backfill toggle) a confident live-search match. A track that + clears none of these is reported unmatched (caller skips it — never a guessed/wrong + id). Returns ``{"resolved": [{artist, title, album, service_track_id}], "stats": + {...}}`` with ``from_cache`` / ``from_library`` / ``from_search`` / ``unmatched`` + tallies for the status display. """ db_fn = db_fn or db_service_track_id total = len(tracks or []) resolved: List[Dict[str, Any]] = [] stats: Dict[str, Any] = { - "total": total, "resolved": 0, "unmatched": 0, "from_cache": 0, "from_library": 0, + "total": total, "resolved": 0, "unmatched": 0, + "from_cache": 0, "from_library": 0, "from_search": 0, } for i, t in enumerate(tracks or []): if not isinstance(t, dict): @@ -192,6 +196,10 @@ def resolve_service_track_ids( tid = db_fn(artist, title, service) if tid: stats["from_library"] += 1 + elif search_id_fn is not None: + tid = search_id_fn(artist, title) + if tid: + stats["from_search"] += 1 resolved.append({"artist": artist, "title": title, "album": album, "service_track_id": tid or None}) @@ -205,6 +213,45 @@ def resolve_service_track_ids( return {"resolved": resolved, "stats": stats} +# Confidence floor for backfill, on the score_track scale (~1.5 = exact title + exact +# artist, thanks to the 1.5x artist boost). A cover/karaoke (x0.05) or a wrong-artist hit +# (no boost, caps ~1.0) can't clear this — so backfill never adds a guessed/wrong version. +BACKFILL_MIN_SCORE = 1.2 + + +def search_service_track_id( + artist: str, + title: str, + *, + search_fn: Callable[[str], List[Any]], + min_score: float = BACKFILL_MIN_SCORE, +) -> Optional[str]: + """Confident live-search match for export backfill (#945): search the target service + for (artist, title), rerank by relevance, and return the top match's id ONLY if it + clears the confidence floor. Below the floor → None: the track is left out of the + export rather than risk a wrong/cover/karaoke version (the whole point of backfill is + coverage WITHOUT the wrong-track risk). ``search_fn(query) -> List[Track]`` is injected + so this is unit-testable without a live service.""" + if not title: + return None + from core.metadata.relevance import build_combined_search_query, filter_and_rerank + query = build_combined_search_query(title, artist) + try: + candidates = list(search_fn(query) or []) + except Exception as exc: + logger.debug(f"export backfill search failed for '{artist} - {title}': {exc}") + return None + if not candidates: + return None + ranked = filter_and_rerank( + candidates, expected_title=title, expected_artist=artist, min_score=min_score, + ) + if not ranked: + return None + tid = getattr(ranked[0], "id", None) + return str(tid) if tid else None + + def file_recording_mbid(artist: str, title: str) -> Optional[str]: """Recording MBID read from the matched track's file tag (set on import post-processing).""" _mbid, fpath = _db_match(artist, title) diff --git a/tests/exports/test_export_sources.py b/tests/exports/test_export_sources.py index 8555903b..3772031d 100644 --- a/tests/exports/test_export_sources.py +++ b/tests/exports/test_export_sources.py @@ -168,4 +168,71 @@ def test_resolve_waterfall_cache_then_library_then_unmatched(): ids = [r['service_track_id'] for r in out['resolved']] assert ids == ['111', 'lib-222', None] s = out['stats'] - assert s == {'total': 3, 'resolved': 2, 'unmatched': 1, 'from_cache': 1, 'from_library': 1} + assert s == {'total': 3, 'resolved': 2, 'unmatched': 1, 'from_cache': 1, + 'from_library': 1, 'from_search': 0} + + +# ── backfill: confident live-search match for the un-cached/un-enriched tail (#945) ── + +from core.metadata.types import Track as _Track +from core.exports.export_sources import search_service_track_id, BACKFILL_MIN_SCORE + + +def _cand(name, artist, tid, album_type='album'): + return _Track(id=tid, name=name, artists=[artist], album='A', + duration_ms=200000, album_type=album_type) + + +def test_backfill_exact_match_returned(): + search = lambda q: [_cand('Not Like Us', 'Kendrick Lamar', 'dz-NLU')] + assert search_service_track_id('Kendrick Lamar', 'Not Like Us', search_fn=search) == 'dz-NLU' + + +def test_backfill_wrong_artist_rejected(): + """SAFETY: an exact-title hit by the WRONG artist scores below the floor (no 1.5x exact- + artist boost) → None, so backfill never adds someone else's same-named track.""" + search = lambda q: [_cand('Not Like Us', 'Some Other Guy', 'wrong-id')] + assert search_service_track_id('Kendrick Lamar', 'Not Like Us', search_fn=search) is None + + +def test_backfill_karaoke_cover_rejected(): + """SAFETY: a karaoke/cover version is buried (x0.05) below the floor → None.""" + search = lambda q: [_cand('Not Like Us (Karaoke Version)', 'Karaoke All Stars', 'kar-id')] + assert search_service_track_id('Kendrick Lamar', 'Not Like Us', search_fn=search) is None + + +def test_backfill_picks_real_over_cover(): + search = lambda q: [ + _cand('Not Like Us (Karaoke Version)', 'Karaoke All Stars', 'kar-id'), + _cand('Not Like Us', 'Kendrick Lamar', 'real-id'), + ] + assert search_service_track_id('Kendrick Lamar', 'Not Like Us', search_fn=search) == 'real-id' + + +def test_backfill_empty_and_error_and_no_title(): + assert search_service_track_id('A', 'X', search_fn=lambda q: []) is None + def boom(q): + raise RuntimeError('deezer flaked') + assert search_service_track_id('A', 'X', search_fn=boom) is None # fail-safe + assert search_service_track_id('A', '', search_fn=lambda q: [_cand('X', 'A', 'i')]) is None + + +def test_resolve_waterfall_uses_search_only_when_cache_and_library_miss(): + tracks = [ + _extra('deezer', 111) | {'artist_name': 'A', 'track_name': 'Cached'}, + {'artist_name': 'A', 'track_name': 'InLib'}, + {'artist_name': 'A', 'track_name': 'OnlyOnSvc'}, + ] + db_fn = lambda a, t, s: 'lib-2' if t == 'InLib' else None + search_id_fn = lambda a, t: 'srch-3' if t == 'OnlyOnSvc' else None + out = resolve_service_track_ids(tracks, 'deezer', db_fn=db_fn, search_id_fn=search_id_fn) + assert [r['service_track_id'] for r in out['resolved']] == ['111', 'lib-2', 'srch-3'] + s = out['stats'] + assert (s['from_cache'], s['from_library'], s['from_search'], s['unmatched']) == (1, 1, 1, 0) + + +def test_resolve_no_search_fn_leaves_tail_unmatched(): + out = resolve_service_track_ids([{'artist_name': 'A', 'track_name': 'X'}], 'deezer', + db_fn=lambda a, t, s: None) # search_id_fn omitted + assert out['resolved'][0]['service_track_id'] is None + assert out['stats']['unmatched'] == 1 and out['stats']['from_search'] == 0 diff --git a/tests/test_service_playlist_export.py b/tests/test_service_playlist_export.py index 790c85b5..f2571aa2 100644 --- a/tests/test_service_playlist_export.py +++ b/tests/test_service_playlist_export.py @@ -29,9 +29,12 @@ class _FakeClient: return self.result -def _fake_resolver(ids): - """resolve_ids_fn stub returning the given service ids as the resolved set.""" - def fn(tracks, service, on_progress=None): +def _fake_resolver(ids, seen=None): + """resolve_ids_fn stub returning the given service ids. When ``seen`` is given, records + the search_id_fn it was called with (to assert the backfill toggle wiring).""" + def fn(tracks, service, search_id_fn=None, on_progress=None): + if seen is not None: + seen['search_id_fn'] = search_id_fn resolved = [{'artist': 'A', 'title': f't{i}', 'service_track_id': s} for i, s in enumerate(ids)] matched = sum(1 for s in ids if s) @@ -88,3 +91,22 @@ def test_reexport_passes_existing_target(): client = _FakeClient({'success': True, 'playlist_id': 'pl-old', 'added': 1}) ws._run_service_export(job, db, 5, 'PL', 'deezer', client, _fake_resolver(['dz'])) assert client.calls[0][2] == 'pl-old' + + +def test_backfill_off_passes_no_search_fn(): + job = {} # no 'backfill' key → off + seen = {} + ws._run_service_export(job, _FakeDB([{}]), 5, 'PL', 'deezer', + _FakeClient({'success': True, 'playlist_id': 'p', 'added': 1}), + _fake_resolver(['dz'], seen)) + assert seen['search_id_fn'] is None + + +def test_backfill_on_wires_search_fn(monkeypatch): + job = {'backfill': True} + seen = {} + monkeypatch.setattr(ws, '_build_service_search_id_fn', lambda service: 'SEARCH_FN') + ws._run_service_export(job, _FakeDB([{}]), 5, 'PL', 'spotify', + _FakeClient({'success': True, 'playlist_id': 'p', 'added': 1}), + _fake_resolver(['sx'], seen)) + assert seen['search_id_fn'] == 'SEARCH_FN' diff --git a/web_server.py b/web_server.py index b555f1b3..49825799 100644 --- a/web_server.py +++ b/web_server.py @@ -27741,14 +27741,34 @@ _playlist_export_jobs = {} _playlist_export_jobs_lock = threading.Lock() +def _build_service_search_id_fn(service): + """Bind a confident-search ID resolver to the service's metadata search client, for the + opt-in export backfill (#945). Returns None when the search client isn't available — the + backfill then simply finds nothing for that track rather than crashing the export.""" + from core.exports.export_sources import search_service_track_id + try: + if service == 'spotify': + client = get_spotify_client() + else: + from core.metadata.registry import get_deezer_client + client = get_deezer_client() + if not client: + return None + search_fn = lambda q: client.search_tracks(q, limit=8) # noqa: E731 + return lambda artist, title: search_service_track_id(artist, title, search_fn=search_fn) + except Exception: + return None + + def _run_service_export(job, db, playlist_id, title, service, client, resolve_ids_fn=None): """Export a mirrored playlist back to a streaming service (Spotify/Deezer, #945). Resolves each track to a target-service ID via the discovery cache (the track's - extra_data — free + already matched) then the library's stored id, pushes the matched - set via the service write client, and stores the returned playlist id so a re-export - updates in place (idempotent, like the LB #903 fix). Deps injected so the orchestration - is unit-testable without a DB or live service. + extra_data — free + already matched) → the library's stored id → (when job['backfill'] + is set) a confident live-search match, pushes the matched set via the service write + client, and stores the returned playlist id so a re-export updates in place (idempotent, + like the LB #903 fix). Deps injected so the orchestration is unit-testable without a DB + or live service. """ from core.exports.export_sources import resolve_service_track_ids resolve_ids_fn = resolve_ids_fn or resolve_service_track_ids @@ -27761,7 +27781,9 @@ def _run_service_export(job, db, playlist_id, title, service, client, resolve_id job['done'] = done job['stats'] = dict(stats) - out = resolve_ids_fn(tracks, service, on_progress=on_progress) + # Opt-in backfill: confident live-search for tracks the cache + library couldn't resolve. + search_id_fn = _build_service_search_id_fn(service) if job.get('backfill') else None + out = resolve_ids_fn(tracks, service, search_id_fn=search_id_fn, on_progress=on_progress) job['stats'] = out['stats'] ids = [r['service_track_id'] for r in out['resolved'] if r.get('service_track_id')] if not ids: @@ -27885,6 +27907,8 @@ def start_playlist_export_service(playlist_id, service): service = (service or '').lower() if service not in ('spotify', 'deezer'): return jsonify({"success": False, "error": f"Unsupported export target: {service}"}), 400 + body = request.get_json(silent=True) or {} + backfill = bool(body.get('backfill')) db = get_database() meta = db.get_mirrored_playlist(int(playlist_id)) or {} title = (meta.get('name') or meta.get('title') or 'SoulSync Export').strip() or 'SoulSync Export' @@ -27894,8 +27918,8 @@ def start_playlist_export_service(playlist_id, service): with _playlist_export_jobs_lock: _playlist_export_jobs[job_id] = { 'job_id': job_id, 'playlist_id': str(playlist_id), 'title': title, - 'mode': service, 'phase': 'starting', 'done': 0, 'total': 0, - 'stats': {}, 'error': None, + 'mode': service, 'backfill': backfill, 'phase': 'starting', 'done': 0, + 'total': 0, 'stats': {}, 'error': None, } t = threading.Thread(target=_run_playlist_export, args=(job_id, playlist_id, title, service), daemon=True) diff --git a/webui/static/stats-automations.js b/webui/static/stats-automations.js index 81e81ca2..61eff33e 100644 --- a/webui/static/stats-automations.js +++ b/webui/static/stats-automations.js @@ -676,20 +676,26 @@ function exportMirroredPlaylist(playlistId, name) {
Create a Deezer playlist from this list (uses your Deezer login).
Tracks are matched by ID (MusicBrainz for ListenBrainz/JSPF; the stored Spotify/Deezer ID for those). Tracks without a match can't be included — you'll see how many made it. Renaming/re-syncing can reset play counts on the destination.
+
`; overlay.addEventListener('click', (e) => { if (e.target === overlay) overlay.remove(); }); overlay.querySelectorAll('.pl-export-choice').forEach(btn => { btn.addEventListener('click', () => { const mode = btn.dataset.mode; + const bfEl = overlay.querySelector('#pl-export-backfill'); + const backfill = !!(bfEl && bfEl.checked); overlay.remove(); - _startPlaylistExport(playlistId, mode, name); + _startPlaylistExport(playlistId, mode, name, backfill); }); }); document.body.appendChild(overlay); } -async function _startPlaylistExport(playlistId, mode, name) { +async function _startPlaylistExport(playlistId, mode, name, backfill) { _setExportStatus(playlistId, `Starting export…`); try { // Spotify/Deezer go to the service endpoint; ListenBrainz/JSPF keep the LB one. @@ -699,7 +705,7 @@ async function _startPlaylistExport(playlistId, mode, name) { : `/api/playlists/${playlistId}/export/listenbrainz`; const resp = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: isService ? '{}' : JSON.stringify({ mode }), + body: isService ? JSON.stringify({ backfill: !!backfill }) : JSON.stringify({ mode }), }); const data = await resp.json(); if (!data.success || !data.job_id) { @@ -729,7 +735,7 @@ async function _pollPlaylistExport(jobId, playlistId, mode, name) { if (mode === 'spotify' || mode === 'deezer') { const dest = mode[0].toUpperCase() + mode.slice(1); const push = job.push || {}; - const cov = `${st.resolved || 0} added${st.unmatched ? ` · ${st.unmatched} not on ${dest}` : ''}`; + const cov = `${st.resolved || 0} added${st.from_search ? ` (${st.from_search} matched live)` : ''}${st.unmatched ? ` · ${st.unmatched} not on ${dest}` : ''}`; const link = push.url ? ` open` : ''; _setExportStatus(playlistId, `Exported to ${dest} · ${cov}${link}`, 12000); if (typeof showToast === 'function') showToast(`Playlist exported to ${dest} (${cov})`, 'success');