playlist export: opt-in confident-search backfill for the unmatched tail (#945, increment 7)
Adds the third resolver stage for tracks the discovery cache + library can't resolve — a live search of the target service, gated behind a "Match missing tracks" toggle so the API cost is opt-in. The whole point is coverage WITHOUT the wrong-track risk, so it's a CONFIDENT match, not "search and grab": - search_service_track_id(artist, title, search_fn): searches the service, reranks via the existing relevance scorer (filter_and_rerank), and returns the top hit's id ONLY if it clears BACKFILL_MIN_SCORE (1.2 on the score_track scale). A wrong-artist hit (no 1.5x exact-artist boost, caps ~1.0) or a karaoke/cover (x0.05) can't clear the floor → None, and the track is left out rather than added wrong. search_fn injected → unit-testable without a live service. - resolve_service_track_ids gains an optional search_id_fn: cache → library → search. Tallies from_search separately. - _run_service_export builds the search fn from the service's metadata search client only when job['backfill'] is set; the endpoint reads `backfill` from the body; the modal adds the toggle and the status line shows "(N matched live)". Store-back of confident matches deferred: a mirrored-only track may have no library row to write to, so persisting needs the track→library mapping — a follow-up, not correctness. 9 new tests incl. the safety ones: wrong-artist rejected, karaoke/cover rejected, real-over-cover picked, fail-safe on search error, and the cache→library→search waterfall + toggle wiring (on/off). 28 export/orchestration tests green, 64 script-integrity green, ruff clean.
This commit is contained in:
BoulderBadgeDad
parent
91eaaeabb2
commit
4743dfd644
5 changed files with 186 additions and 20 deletions
|
|
@ -162,21 +162,25 @@ def resolve_service_track_ids(
|
|||
service: str,
|
||||
*,
|
||||
db_fn: Optional[Callable[[str, str, str], Optional[str]]] = None,
|
||||
search_id_fn: Optional[Callable[[str, str], Optional[str]]] = None,
|
||||
on_progress: Optional[Callable[[int, int, Dict[str, Any]], None]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Resolve a mirrored playlist's tracks to target-service track IDs for export.
|
||||
|
||||
Waterfall per track: the discovery cache (``extra_data`` — free + already confidently
|
||||
matched) → the library track's stored service id. A track with neither is reported
|
||||
unmatched (caller skips it — never a guessed/wrong id). Returns
|
||||
``{"resolved": [{artist, title, album, service_track_id}], "stats": {...}}`` with
|
||||
``from_cache`` / ``from_library`` / ``unmatched`` tallies for the status display.
|
||||
matched) → the library track's stored service id → (only when ``search_id_fn`` is
|
||||
given, i.e. the opt-in backfill toggle) a confident live-search match. A track that
|
||||
clears none of these is reported unmatched (caller skips it — never a guessed/wrong
|
||||
id). Returns ``{"resolved": [{artist, title, album, service_track_id}], "stats":
|
||||
{...}}`` with ``from_cache`` / ``from_library`` / ``from_search`` / ``unmatched``
|
||||
tallies for the status display.
|
||||
"""
|
||||
db_fn = db_fn or db_service_track_id
|
||||
total = len(tracks or [])
|
||||
resolved: List[Dict[str, Any]] = []
|
||||
stats: Dict[str, Any] = {
|
||||
"total": total, "resolved": 0, "unmatched": 0, "from_cache": 0, "from_library": 0,
|
||||
"total": total, "resolved": 0, "unmatched": 0,
|
||||
"from_cache": 0, "from_library": 0, "from_search": 0,
|
||||
}
|
||||
for i, t in enumerate(tracks or []):
|
||||
if not isinstance(t, dict):
|
||||
|
|
@ -192,6 +196,10 @@ def resolve_service_track_ids(
|
|||
tid = db_fn(artist, title, service)
|
||||
if tid:
|
||||
stats["from_library"] += 1
|
||||
elif search_id_fn is not None:
|
||||
tid = search_id_fn(artist, title)
|
||||
if tid:
|
||||
stats["from_search"] += 1
|
||||
|
||||
resolved.append({"artist": artist, "title": title, "album": album,
|
||||
"service_track_id": tid or None})
|
||||
|
|
@ -205,6 +213,45 @@ def resolve_service_track_ids(
|
|||
return {"resolved": resolved, "stats": stats}
|
||||
|
||||
|
||||
# Confidence floor for backfill, on the score_track scale (~1.5 = exact title + exact
|
||||
# artist, thanks to the 1.5x artist boost). A cover/karaoke (x0.05) or a wrong-artist hit
|
||||
# (no boost, caps ~1.0) can't clear this — so backfill never adds a guessed/wrong version.
|
||||
BACKFILL_MIN_SCORE = 1.2
|
||||
|
||||
|
||||
def search_service_track_id(
|
||||
artist: str,
|
||||
title: str,
|
||||
*,
|
||||
search_fn: Callable[[str], List[Any]],
|
||||
min_score: float = BACKFILL_MIN_SCORE,
|
||||
) -> Optional[str]:
|
||||
"""Confident live-search match for export backfill (#945): search the target service
|
||||
for (artist, title), rerank by relevance, and return the top match's id ONLY if it
|
||||
clears the confidence floor. Below the floor → None: the track is left out of the
|
||||
export rather than risk a wrong/cover/karaoke version (the whole point of backfill is
|
||||
coverage WITHOUT the wrong-track risk). ``search_fn(query) -> List[Track]`` is injected
|
||||
so this is unit-testable without a live service."""
|
||||
if not title:
|
||||
return None
|
||||
from core.metadata.relevance import build_combined_search_query, filter_and_rerank
|
||||
query = build_combined_search_query(title, artist)
|
||||
try:
|
||||
candidates = list(search_fn(query) or [])
|
||||
except Exception as exc:
|
||||
logger.debug(f"export backfill search failed for '{artist} - {title}': {exc}")
|
||||
return None
|
||||
if not candidates:
|
||||
return None
|
||||
ranked = filter_and_rerank(
|
||||
candidates, expected_title=title, expected_artist=artist, min_score=min_score,
|
||||
)
|
||||
if not ranked:
|
||||
return None
|
||||
tid = getattr(ranked[0], "id", None)
|
||||
return str(tid) if tid else None
|
||||
|
||||
|
||||
def file_recording_mbid(artist: str, title: str) -> Optional[str]:
|
||||
"""Recording MBID read from the matched track's file tag (set on import post-processing)."""
|
||||
_mbid, fpath = _db_match(artist, title)
|
||||
|
|
|
|||
|
|
@ -168,4 +168,71 @@ def test_resolve_waterfall_cache_then_library_then_unmatched():
|
|||
ids = [r['service_track_id'] for r in out['resolved']]
|
||||
assert ids == ['111', 'lib-222', None]
|
||||
s = out['stats']
|
||||
assert s == {'total': 3, 'resolved': 2, 'unmatched': 1, 'from_cache': 1, 'from_library': 1}
|
||||
assert s == {'total': 3, 'resolved': 2, 'unmatched': 1, 'from_cache': 1,
|
||||
'from_library': 1, 'from_search': 0}
|
||||
|
||||
|
||||
# ── backfill: confident live-search match for the un-cached/un-enriched tail (#945) ──
|
||||
|
||||
from core.metadata.types import Track as _Track
|
||||
from core.exports.export_sources import search_service_track_id, BACKFILL_MIN_SCORE
|
||||
|
||||
|
||||
def _cand(name, artist, tid, album_type='album'):
|
||||
return _Track(id=tid, name=name, artists=[artist], album='A',
|
||||
duration_ms=200000, album_type=album_type)
|
||||
|
||||
|
||||
def test_backfill_exact_match_returned():
|
||||
search = lambda q: [_cand('Not Like Us', 'Kendrick Lamar', 'dz-NLU')]
|
||||
assert search_service_track_id('Kendrick Lamar', 'Not Like Us', search_fn=search) == 'dz-NLU'
|
||||
|
||||
|
||||
def test_backfill_wrong_artist_rejected():
|
||||
"""SAFETY: an exact-title hit by the WRONG artist scores below the floor (no 1.5x exact-
|
||||
artist boost) → None, so backfill never adds someone else's same-named track."""
|
||||
search = lambda q: [_cand('Not Like Us', 'Some Other Guy', 'wrong-id')]
|
||||
assert search_service_track_id('Kendrick Lamar', 'Not Like Us', search_fn=search) is None
|
||||
|
||||
|
||||
def test_backfill_karaoke_cover_rejected():
|
||||
"""SAFETY: a karaoke/cover version is buried (x0.05) below the floor → None."""
|
||||
search = lambda q: [_cand('Not Like Us (Karaoke Version)', 'Karaoke All Stars', 'kar-id')]
|
||||
assert search_service_track_id('Kendrick Lamar', 'Not Like Us', search_fn=search) is None
|
||||
|
||||
|
||||
def test_backfill_picks_real_over_cover():
|
||||
search = lambda q: [
|
||||
_cand('Not Like Us (Karaoke Version)', 'Karaoke All Stars', 'kar-id'),
|
||||
_cand('Not Like Us', 'Kendrick Lamar', 'real-id'),
|
||||
]
|
||||
assert search_service_track_id('Kendrick Lamar', 'Not Like Us', search_fn=search) == 'real-id'
|
||||
|
||||
|
||||
def test_backfill_empty_and_error_and_no_title():
|
||||
assert search_service_track_id('A', 'X', search_fn=lambda q: []) is None
|
||||
def boom(q):
|
||||
raise RuntimeError('deezer flaked')
|
||||
assert search_service_track_id('A', 'X', search_fn=boom) is None # fail-safe
|
||||
assert search_service_track_id('A', '', search_fn=lambda q: [_cand('X', 'A', 'i')]) is None
|
||||
|
||||
|
||||
def test_resolve_waterfall_uses_search_only_when_cache_and_library_miss():
|
||||
tracks = [
|
||||
_extra('deezer', 111) | {'artist_name': 'A', 'track_name': 'Cached'},
|
||||
{'artist_name': 'A', 'track_name': 'InLib'},
|
||||
{'artist_name': 'A', 'track_name': 'OnlyOnSvc'},
|
||||
]
|
||||
db_fn = lambda a, t, s: 'lib-2' if t == 'InLib' else None
|
||||
search_id_fn = lambda a, t: 'srch-3' if t == 'OnlyOnSvc' else None
|
||||
out = resolve_service_track_ids(tracks, 'deezer', db_fn=db_fn, search_id_fn=search_id_fn)
|
||||
assert [r['service_track_id'] for r in out['resolved']] == ['111', 'lib-2', 'srch-3']
|
||||
s = out['stats']
|
||||
assert (s['from_cache'], s['from_library'], s['from_search'], s['unmatched']) == (1, 1, 1, 0)
|
||||
|
||||
|
||||
def test_resolve_no_search_fn_leaves_tail_unmatched():
|
||||
out = resolve_service_track_ids([{'artist_name': 'A', 'track_name': 'X'}], 'deezer',
|
||||
db_fn=lambda a, t, s: None) # search_id_fn omitted
|
||||
assert out['resolved'][0]['service_track_id'] is None
|
||||
assert out['stats']['unmatched'] == 1 and out['stats']['from_search'] == 0
|
||||
|
|
|
|||
|
|
@ -29,9 +29,12 @@ class _FakeClient:
|
|||
return self.result
|
||||
|
||||
|
||||
def _fake_resolver(ids):
|
||||
"""resolve_ids_fn stub returning the given service ids as the resolved set."""
|
||||
def fn(tracks, service, on_progress=None):
|
||||
def _fake_resolver(ids, seen=None):
|
||||
"""resolve_ids_fn stub returning the given service ids. When ``seen`` is given, records
|
||||
the search_id_fn it was called with (to assert the backfill toggle wiring)."""
|
||||
def fn(tracks, service, search_id_fn=None, on_progress=None):
|
||||
if seen is not None:
|
||||
seen['search_id_fn'] = search_id_fn
|
||||
resolved = [{'artist': 'A', 'title': f't{i}', 'service_track_id': s}
|
||||
for i, s in enumerate(ids)]
|
||||
matched = sum(1 for s in ids if s)
|
||||
|
|
@ -88,3 +91,22 @@ def test_reexport_passes_existing_target():
|
|||
client = _FakeClient({'success': True, 'playlist_id': 'pl-old', 'added': 1})
|
||||
ws._run_service_export(job, db, 5, 'PL', 'deezer', client, _fake_resolver(['dz']))
|
||||
assert client.calls[0][2] == 'pl-old'
|
||||
|
||||
|
||||
def test_backfill_off_passes_no_search_fn():
|
||||
job = {} # no 'backfill' key → off
|
||||
seen = {}
|
||||
ws._run_service_export(job, _FakeDB([{}]), 5, 'PL', 'deezer',
|
||||
_FakeClient({'success': True, 'playlist_id': 'p', 'added': 1}),
|
||||
_fake_resolver(['dz'], seen))
|
||||
assert seen['search_id_fn'] is None
|
||||
|
||||
|
||||
def test_backfill_on_wires_search_fn(monkeypatch):
|
||||
job = {'backfill': True}
|
||||
seen = {}
|
||||
monkeypatch.setattr(ws, '_build_service_search_id_fn', lambda service: 'SEARCH_FN')
|
||||
ws._run_service_export(job, _FakeDB([{}]), 5, 'PL', 'spotify',
|
||||
_FakeClient({'success': True, 'playlist_id': 'p', 'added': 1}),
|
||||
_fake_resolver(['sx'], seen))
|
||||
assert seen['search_id_fn'] == 'SEARCH_FN'
|
||||
|
|
|
|||
|
|
@ -27741,14 +27741,34 @@ _playlist_export_jobs = {}
|
|||
_playlist_export_jobs_lock = threading.Lock()
|
||||
|
||||
|
||||
def _build_service_search_id_fn(service):
|
||||
"""Bind a confident-search ID resolver to the service's metadata search client, for the
|
||||
opt-in export backfill (#945). Returns None when the search client isn't available — the
|
||||
backfill then simply finds nothing for that track rather than crashing the export."""
|
||||
from core.exports.export_sources import search_service_track_id
|
||||
try:
|
||||
if service == 'spotify':
|
||||
client = get_spotify_client()
|
||||
else:
|
||||
from core.metadata.registry import get_deezer_client
|
||||
client = get_deezer_client()
|
||||
if not client:
|
||||
return None
|
||||
search_fn = lambda q: client.search_tracks(q, limit=8) # noqa: E731
|
||||
return lambda artist, title: search_service_track_id(artist, title, search_fn=search_fn)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _run_service_export(job, db, playlist_id, title, service, client, resolve_ids_fn=None):
|
||||
"""Export a mirrored playlist back to a streaming service (Spotify/Deezer, #945).
|
||||
|
||||
Resolves each track to a target-service ID via the discovery cache (the track's
|
||||
extra_data — free + already matched) then the library's stored id, pushes the matched
|
||||
set via the service write client, and stores the returned playlist id so a re-export
|
||||
updates in place (idempotent, like the LB #903 fix). Deps injected so the orchestration
|
||||
is unit-testable without a DB or live service.
|
||||
extra_data — free + already matched) → the library's stored id → (when job['backfill']
|
||||
is set) a confident live-search match, pushes the matched set via the service write
|
||||
client, and stores the returned playlist id so a re-export updates in place (idempotent,
|
||||
like the LB #903 fix). Deps injected so the orchestration is unit-testable without a DB
|
||||
or live service.
|
||||
"""
|
||||
from core.exports.export_sources import resolve_service_track_ids
|
||||
resolve_ids_fn = resolve_ids_fn or resolve_service_track_ids
|
||||
|
|
@ -27761,7 +27781,9 @@ def _run_service_export(job, db, playlist_id, title, service, client, resolve_id
|
|||
job['done'] = done
|
||||
job['stats'] = dict(stats)
|
||||
|
||||
out = resolve_ids_fn(tracks, service, on_progress=on_progress)
|
||||
# Opt-in backfill: confident live-search for tracks the cache + library couldn't resolve.
|
||||
search_id_fn = _build_service_search_id_fn(service) if job.get('backfill') else None
|
||||
out = resolve_ids_fn(tracks, service, search_id_fn=search_id_fn, on_progress=on_progress)
|
||||
job['stats'] = out['stats']
|
||||
ids = [r['service_track_id'] for r in out['resolved'] if r.get('service_track_id')]
|
||||
if not ids:
|
||||
|
|
@ -27885,6 +27907,8 @@ def start_playlist_export_service(playlist_id, service):
|
|||
service = (service or '').lower()
|
||||
if service not in ('spotify', 'deezer'):
|
||||
return jsonify({"success": False, "error": f"Unsupported export target: {service}"}), 400
|
||||
body = request.get_json(silent=True) or {}
|
||||
backfill = bool(body.get('backfill'))
|
||||
db = get_database()
|
||||
meta = db.get_mirrored_playlist(int(playlist_id)) or {}
|
||||
title = (meta.get('name') or meta.get('title') or 'SoulSync Export').strip() or 'SoulSync Export'
|
||||
|
|
@ -27894,8 +27918,8 @@ def start_playlist_export_service(playlist_id, service):
|
|||
with _playlist_export_jobs_lock:
|
||||
_playlist_export_jobs[job_id] = {
|
||||
'job_id': job_id, 'playlist_id': str(playlist_id), 'title': title,
|
||||
'mode': service, 'phase': 'starting', 'done': 0, 'total': 0,
|
||||
'stats': {}, 'error': None,
|
||||
'mode': service, 'backfill': backfill, 'phase': 'starting', 'done': 0,
|
||||
'total': 0, 'stats': {}, 'error': None,
|
||||
}
|
||||
t = threading.Thread(target=_run_playlist_export,
|
||||
args=(job_id, playlist_id, title, service), daemon=True)
|
||||
|
|
|
|||
|
|
@ -676,20 +676,26 @@ function exportMirroredPlaylist(playlistId, name) {
|
|||
<div style="font-size:12px;color:rgba(255,255,255,0.55);">Create a Deezer playlist from this list (uses your Deezer login).</div>
|
||||
</button>
|
||||
<div style="font-size:11.5px;color:rgba(255,255,255,0.4);line-height:1.5;">Tracks are matched by ID (MusicBrainz for ListenBrainz/JSPF; the stored Spotify/Deezer ID for those). Tracks without a match can't be included — you'll see how many made it. Renaming/re-syncing can reset play counts on the destination.</div>
|
||||
<label style="display:flex;align-items:flex-start;gap:8px;margin-top:12px;font-size:12px;color:rgba(255,255,255,0.6);cursor:pointer;">
|
||||
<input type="checkbox" id="pl-export-backfill" style="margin-top:2px;flex-shrink:0;accent-color:rgb(var(--accent-rgb));">
|
||||
<span><b style="color:rgba(255,255,255,0.75);">Match missing tracks</b> (Spotify/Deezer only) — search the service for tracks with no known ID. Slower, and only confident matches are added.</span>
|
||||
</label>
|
||||
<div style="text-align:right;margin-top:14px;"><button onclick="document.getElementById('pl-export-modal').remove()" style="background:none;border:none;color:rgba(255,255,255,0.5);cursor:pointer;font-size:13px;">Cancel</button></div>
|
||||
</div>`;
|
||||
overlay.addEventListener('click', (e) => { if (e.target === overlay) overlay.remove(); });
|
||||
overlay.querySelectorAll('.pl-export-choice').forEach(btn => {
|
||||
btn.addEventListener('click', () => {
|
||||
const mode = btn.dataset.mode;
|
||||
const bfEl = overlay.querySelector('#pl-export-backfill');
|
||||
const backfill = !!(bfEl && bfEl.checked);
|
||||
overlay.remove();
|
||||
_startPlaylistExport(playlistId, mode, name);
|
||||
_startPlaylistExport(playlistId, mode, name, backfill);
|
||||
});
|
||||
});
|
||||
document.body.appendChild(overlay);
|
||||
}
|
||||
|
||||
async function _startPlaylistExport(playlistId, mode, name) {
|
||||
async function _startPlaylistExport(playlistId, mode, name, backfill) {
|
||||
_setExportStatus(playlistId, `<span style="color:#a78bfa;">Starting export…</span>`);
|
||||
try {
|
||||
// Spotify/Deezer go to the service endpoint; ListenBrainz/JSPF keep the LB one.
|
||||
|
|
@ -699,7 +705,7 @@ async function _startPlaylistExport(playlistId, mode, name) {
|
|||
: `/api/playlists/${playlistId}/export/listenbrainz`;
|
||||
const resp = await fetch(url, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: isService ? '{}' : JSON.stringify({ mode }),
|
||||
body: isService ? JSON.stringify({ backfill: !!backfill }) : JSON.stringify({ mode }),
|
||||
});
|
||||
const data = await resp.json();
|
||||
if (!data.success || !data.job_id) {
|
||||
|
|
@ -729,7 +735,7 @@ async function _pollPlaylistExport(jobId, playlistId, mode, name) {
|
|||
if (mode === 'spotify' || mode === 'deezer') {
|
||||
const dest = mode[0].toUpperCase() + mode.slice(1);
|
||||
const push = job.push || {};
|
||||
const cov = `${st.resolved || 0} added${st.unmatched ? ` · ${st.unmatched} not on ${dest}` : ''}`;
|
||||
const cov = `${st.resolved || 0} added${st.from_search ? ` (${st.from_search} matched live)` : ''}${st.unmatched ? ` · ${st.unmatched} not on ${dest}` : ''}`;
|
||||
const link = push.url ? ` <a href="${push.url}" target="_blank" style="color:#38bdf8;">open</a>` : '';
|
||||
_setExportStatus(playlistId, `<span style="color:#22c55e;">Exported to ${dest} · ${cov}</span>${link}`, 12000);
|
||||
if (typeof showToast === 'function') showToast(`Playlist exported to ${dest} (${cov})`, 'success');
|
||||
|
|
|
|||
Loading…
Reference in a new issue