From de20897f833302c9b01210e379bd654c681c2e06 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Mon, 1 Jun 2026 16:30:28 -0700 Subject: [PATCH 001/108] Fix: deep-scan / DB-update automation falsely errors on large libraries (stall-based timeout) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DB-update + deep-scan automation monitor used a hard 2-hour TOTAL cap (while elapsed < 7200). It tracked progress but only used it to print a stall warning — the only thing that actually timed out was wall-clock. So a large library that scans for >2h while progressing fine (reported: 4781 artists) trips the cap and the automation card flips to 'error: timed out after 2 hours' even though the scan thread is healthy and still running (the timeout never cancels it, which is why it keeps progressing in the logs after the 'error'). Time out on STALL, not total runtime: - 30 min with NO progress -> error ('stalled'); catches a genuinely hung scan. - 10 min idle -> warning (repeats); unchanged heads-up. - 24h absolute backstop, purely a runaway-loop guard. - An actively-progressing scan keeps resetting the idle clock, so it never times out no matter how many hours the whole library takes. - Progress is judged on (processed, progress, current_item) so a slow stretch where the rounded % holds steady (but the artist keeps changing) isn't a false stall. The decision is extracted into a pure, testable scan_wait_action(); both the deep-scan and full-refresh handlers share the monitor loop, so both are fixed. Tests: tests/test_scan_wait_action.py (9) — headline regression (5h/12h total but progressing -> 'continue', not timeout), finished/stall-warn/stall-timeout/ abs-cap thresholds, and ordering. 280 automation tests still pass. --- core/automation/handlers/database_update.py | 95 +++++++++++++++++---- tests/test_scan_wait_action.py | 84 ++++++++++++++++++ 2 files changed, 162 insertions(+), 17 deletions(-) create mode 100644 tests/test_scan_wait_action.py diff --git a/core/automation/handlers/database_update.py b/core/automation/handlers/database_update.py index a9d51036..d123be24 100644 --- a/core/automation/handlers/database_update.py +++ b/core/automation/handlers/database_update.py @@ -21,12 +21,49 @@ from typing import Any, Dict from core.automation.deps import AutomationDeps -_TIMEOUT_SECONDS = 7200 # 2 hours — covers the worst large-library case -_STALL_WARNING_SECONDS = 600 # 10 minutes without progress = stall +# Time out on STALL (no progress), not total runtime: a large library can scan +# for many hours while progressing fine — a hard total cap would falsely mark a +# healthy scan 'error' (the scan thread keeps running uncancelled). We only give +# up when progress hasn't moved for a long stretch, with a generous absolute +# backstop against a truly stuck monitor loop. +_STALL_WARNING_SECONDS = 600 # warn after 10 min with no progress (repeats) +_STALL_TIMEOUT_SECONDS = 1800 # 30 min with no progress at all = genuinely stalled +_ABSOLUTE_CAP_SECONDS = 86400 # 24h hard backstop (runaway-loop guard only) _POLL_INTERVAL_SECONDS = 3 _INITIAL_DELAY_SECONDS = 1 +def scan_wait_action( + *, + status: str, + idle_seconds: float, + total_seconds: float, + stall_timeout_s: float = _STALL_TIMEOUT_SECONDS, + stall_warn_s: float = _STALL_WARNING_SECONDS, + abs_cap_s: float = _ABSOLUTE_CAP_SECONDS, +) -> str: + """Decide what the monitor loop should do on a poll tick (pure/testable). + + ``idle_seconds`` is time since progress last changed; ``total_seconds`` is + time since the wait began. Returns one of: + ``'finished'`` (task no longer running), ``'stall_timeout'`` (no progress for + too long → give up), ``'abs_timeout'`` (absolute backstop), ``'warn'`` + (stalled long enough to warn but not give up), or ``'continue'``. + + Crucially, an actively-progressing scan keeps resetting ``idle_seconds``, so + it never hits ``stall_timeout`` no matter how long the whole scan takes. + """ + if status != 'running': + return 'finished' + if total_seconds >= abs_cap_s: + return 'abs_timeout' + if idle_seconds >= stall_timeout_s: + return 'stall_timeout' + if idle_seconds >= stall_warn_s: + return 'warn' + return 'continue' + + def auto_start_database_update(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]: """Run a full or incremental DB update via ``run_db_update_task``.""" return _run_with_progress( @@ -36,7 +73,7 @@ def auto_start_database_update(config: Dict[str, Any], deps: AutomationDeps) -> initial_phase='Initializing...', stall_label='Database update', finished_extras=lambda: {'full_refresh': str(config.get('full_refresh', False))}, - timeout_label='Database update timed out after 2 hours', + timeout_label='Database update timed out after 24 hours', ) @@ -49,7 +86,7 @@ def auto_deep_scan_library(config: Dict[str, Any], deps: AutomationDeps) -> Dict initial_phase='Deep scan: Initializing...', stall_label='Deep scan', finished_extras=lambda: {}, - timeout_label='Deep scan timed out after 2 hours', + timeout_label='Deep scan timed out after 24 hours', ) @@ -83,30 +120,54 @@ def _run_with_progress( deps.db_update_executor.submit(task, *task_args) # Monitor progress (callbacks handle card updates, we just block until done). + # We time out on STALL, not total runtime: ``processed`` advances on every + # artist, so an actively-progressing scan keeps resetting the idle clock and + # is never falsely failed no matter how long the whole library takes. time.sleep(_INITIAL_DELAY_SECONDS) poll_start = time.time() last_progress_time = time.time() - last_progress_val = 0 - while time.time() - poll_start < _TIMEOUT_SECONDS: + # Any of these advancing means the scan is alive. current_item (the artist + # being processed) changes every artist even when the rounded progress % + # holds steady, so it guards against a false stall during slow stretches. + last_progress_val = (0, 0, '') + last_warn_time = 0.0 + outcome = 'finished' + while True: time.sleep(_POLL_INTERVAL_SECONDS) + now = time.time() with deps.db_update_lock: current_status = state.get('status', 'idle') - current_progress = state.get('progress', 0) - if current_status != 'running': + current_val = (state.get('processed', 0), state.get('progress', 0), + state.get('current_item', '')) + if current_val != last_progress_val: + last_progress_val = current_val + last_progress_time = now + + action = scan_wait_action( + status=current_status, + idle_seconds=now - last_progress_time, + total_seconds=now - poll_start, + ) + if action in ('finished', 'stall_timeout', 'abs_timeout'): + outcome = action break - # Stall detection — if no progress change in 10 minutes, warn. - if current_progress != last_progress_val: - last_progress_val = current_progress - last_progress_time = time.time() - elif time.time() - last_progress_time > _STALL_WARNING_SECONDS: + if action == 'warn' and (now - last_warn_time) > _STALL_WARNING_SECONDS: + idle_min = int((now - last_progress_time) / 60) deps.update_progress( automation_id, - log_line=f'{stall_label} appears stalled — waiting...', + log_line=f'{stall_label} — no progress for {idle_min} min, still waiting...', log_type='warning', ) - last_progress_time = time.time() # Reset so warning repeats every 10 min. - else: - # 2-hour timeout reached. + last_warn_time = now + + if outcome == 'stall_timeout': + deps.update_progress( + automation_id, status='error', phase='Stalled', + log_line=f'{stall_label} made no progress for {_STALL_TIMEOUT_SECONDS // 60} minutes — giving up', + log_type='error', + ) + return {'status': 'error', 'reason': 'Stalled (no progress)', '_manages_own_progress': True} + if outcome == 'abs_timeout': deps.update_progress( automation_id, status='error', phase='Timed out', log_line=timeout_label, log_type='error', diff --git a/tests/test_scan_wait_action.py b/tests/test_scan_wait_action.py new file mode 100644 index 00000000..1f3ac004 --- /dev/null +++ b/tests/test_scan_wait_action.py @@ -0,0 +1,84 @@ +"""Tests for the DB-update / deep-scan monitor decision (stall-based timeout). + +Regression: a large library can deep-scan for many hours while progressing +fine. The old monitor used a hard 2-hour TOTAL cap, so it falsely marked a +healthy, still-running scan 'error' (the scan thread kept going uncancelled). +The decision now keys off STALL (no progress), so an actively-progressing scan +never times out no matter how long the whole library takes. +""" + +from __future__ import annotations + +from core.automation.handlers.database_update import ( + scan_wait_action, + _STALL_WARNING_SECONDS, + _STALL_TIMEOUT_SECONDS, + _ABSOLUTE_CAP_SECONDS, +) + + +# --- the headline regression ------------------------------------------------- + + +def test_long_but_progressing_scan_never_times_out(): + # 5 hours elapsed total, but progress moved 5s ago -> keep waiting, NOT error. + assert scan_wait_action( + status='running', idle_seconds=5, total_seconds=5 * 3600, + ) == 'continue' + + +def test_very_long_progressing_scan_still_continues(): + # 12h total, just progressed — old code would have failed at 2h. + assert scan_wait_action( + status='running', idle_seconds=2, total_seconds=12 * 3600, + ) == 'continue' + + +# --- finished / not-running -------------------------------------------------- + + +def test_finished_when_not_running(): + for st in ('completed', 'error', 'idle', 'finished'): + assert scan_wait_action(status=st, idle_seconds=0, total_seconds=0) == 'finished' + + +def test_finished_takes_precedence_even_if_stalled(): + # Task already ended — don't report a stall. + assert scan_wait_action( + status='completed', idle_seconds=_STALL_TIMEOUT_SECONDS + 1, total_seconds=10, + ) == 'finished' + + +# --- stall warning vs stall timeout ------------------------------------------ + + +def test_warns_after_stall_warning_threshold(): + assert scan_wait_action( + status='running', idle_seconds=_STALL_WARNING_SECONDS + 1, total_seconds=1000, + ) == 'warn' + + +def test_stall_timeout_after_no_progress(): + assert scan_wait_action( + status='running', idle_seconds=_STALL_TIMEOUT_SECONDS + 1, total_seconds=2000, + ) == 'stall_timeout' + + +def test_just_below_warning_keeps_going(): + assert scan_wait_action( + status='running', idle_seconds=_STALL_WARNING_SECONDS - 1, total_seconds=1000, + ) == 'continue' + + +# --- absolute backstop ------------------------------------------------------- + + +def test_absolute_cap_is_last_resort(): + # Even if somehow progressing, a 24h+ wait trips the runaway-loop backstop. + assert scan_wait_action( + status='running', idle_seconds=1, total_seconds=_ABSOLUTE_CAP_SECONDS + 1, + ) == 'abs_timeout' + + +def test_thresholds_are_ordered_sensibly(): + assert _STALL_WARNING_SECONDS < _STALL_TIMEOUT_SECONDS < _ABSOLUTE_CAP_SECONDS From 3dfec8a157f00f518c3fa255218bcb54733b4c3a Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 08:40:05 -0700 Subject: [PATCH 002/108] Fix #764: import no longer destroys embedded cover art MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit enhance_file_metadata rebuilds tags from scratch: for FLAC it calls clear_pictures(), for MP3/MP4 it clears the whole tag block — and it does this UP FRONT, then saves the file, long before it tries to fetch and embed the replacement art. So every way the re-embed could come up empty left the file saved with the original art destroyed and nothing put back: - extract_source_metadata returns nothing -> early save, no embed - no album-art URL / art download fails / rejected by the min-size guard -> embed_album_art_metadata returns early without adding a picture - art embedding disabled in config -> embed skipped entirely - embed raises mid-enrichment -> file left cleared on disk This is the "cover art gets corrupted/destroyed during import" half of #764 (continuation of #755); distinct from #750's truncated-cache DISPLAY bug. Fix: new core/metadata/art_preservation.py snapshots the existing art (the live Picture / APIC / MP4Cover objects, so they re-apply verbatim) BEFORE the clear, and restores it before each save IFF the file currently has none. Wired into all three exit paths in enhance_file_metadata (no-metadata early return, the final save, and the except handler). The restore is a strict no-op when art is already present, so the happy path — new art embedded — is byte-for-byte unchanged: it never clobbers or duplicates a freshly-embedded cover. embed_album_art_metadata now returns a bool so the intent (embedded / didn't) is explicit. Tests: - tests/test_art_preservation.py (5) — snapshot/restore round-trips through real mutagen FLAC + ID3 objects; restore no-ops when new art is present. - tests/test_enrichment_art_preservation.py (4) — runs the REAL enhance_file_metadata over a real FLAC with embedded art and asserts the art survives on disk for missing-metadata / failed-embed / embed-raises, and is correctly REPLACED (exactly one picture, new bytes) on success. 1019 tests pass across the metadata/enrichment/imports/acoustid suites. --- core/metadata/art_preservation.py | 115 ++++++++++++++ core/metadata/artwork.py | 8 +- core/metadata/enrichment.py | 37 +++++ tests/test_art_preservation.py | 179 ++++++++++++++++++++++ tests/test_enrichment_art_preservation.py | 153 ++++++++++++++++++ 5 files changed, 489 insertions(+), 3 deletions(-) create mode 100644 core/metadata/art_preservation.py create mode 100644 tests/test_art_preservation.py create mode 100644 tests/test_enrichment_art_preservation.py diff --git a/core/metadata/art_preservation.py b/core/metadata/art_preservation.py new file mode 100644 index 00000000..337cf7f6 --- /dev/null +++ b/core/metadata/art_preservation.py @@ -0,0 +1,115 @@ +"""Preserve embedded cover art across the metadata-enrichment rewrite. + +Issue #764 (continuation of #755): imported files lost their album art. +``enhance_file_metadata`` rebuilds tags from scratch — for FLAC it calls +``clear_pictures()`` and for MP3/MP4 it clears the whole tag block — *before* +it has the replacement art in hand. It then saves the file regardless of +whether new art was actually embedded. So every failure mode downstream +destroyed the art that shipped with the download: + + - source-metadata extraction returns nothing -> early save, no embed + - no album-art URL available / art download fails -> embed returns early + - art rejected by the min-resolution guard -> embed returns early + - art embedding disabled in config -> embed skipped entirely + +In all of those the file was saved with the pictures already cleared and +nothing put back. This module captures the existing art up front (live +mutagen objects, so they re-apply verbatim) and restores it right before a +save *iff the file currently has none* — so the happy path (new art embedded) +is byte-for-byte unchanged, and the only behaviour change is that we never +end up with less art than we started with. + +Scope mirrors ``embed_album_art_metadata``: FLAC ``Picture`` blocks, ID3 +``APIC`` frames, MP4 ``covr`` atoms. OggOpus/OggVorbis store art inside the +Vorbis comment (no ``clear_pictures``), so the enrichment rewrite never +strips it and it needs no preservation here. +""" + +from __future__ import annotations + +from typing import Any, List, Tuple + +from utils.logging_config import get_logger + +logger = get_logger("metadata.art_preservation") + +# Each snapshot entry is (kind, payload) where payload is a list of live +# mutagen objects captured before the tag rewrite. +ArtSnapshot = List[Tuple[str, list]] + + +def has_embedded_art(audio_file: Any, symbols: Any) -> bool: + """True iff ``audio_file`` currently carries embedded cover art in any + of the formats the enricher manages (FLAC pictures / ID3 APIC / MP4 covr).""" + try: + if getattr(audio_file, "pictures", None): + return True + tags = getattr(audio_file, "tags", None) + if tags is not None and isinstance(tags, symbols.ID3) and tags.getall("APIC"): + return True + if isinstance(audio_file, symbols.MP4) and tags and tags.get("covr"): + return True + except Exception as exc: # defensive: never let art-detection break a save + logger.debug("has_embedded_art check failed: %s", exc) + return False + + +def snapshot_embedded_art(audio_file: Any, symbols: Any) -> ArtSnapshot: + """Capture existing embedded art so it can be restored if re-embedding + fails. Returns a list of ``(kind, [objects])`` entries, or ``[]`` when the + file has no art. Captures the live mutagen objects (Picture / APIC frame / + MP4Cover) so they re-apply exactly as they were. + + Must be called BEFORE ``clear_pictures()`` / ``tags.clear()``.""" + snap: ArtSnapshot = [] + try: + pictures = getattr(audio_file, "pictures", None) + if pictures: + snap.append(("flac", list(pictures))) + tags = getattr(audio_file, "tags", None) + if tags is not None and isinstance(tags, symbols.ID3): + apics = tags.getall("APIC") + if apics: + snap.append(("id3", list(apics))) + if isinstance(audio_file, symbols.MP4) and tags: + covr = tags.get("covr") + if covr: + snap.append(("mp4", list(covr))) + except Exception as exc: + logger.debug("snapshot_embedded_art failed: %s", exc) + return snap + + +def restore_embedded_art(audio_file: Any, symbols: Any, snapshot: ArtSnapshot) -> bool: + """Re-apply captured art IFF the file currently has none. Returns True if + it restored something. + + No-op (returns False) when the snapshot is empty or the file already has + art — so calling this before a save never overwrites freshly-embedded art, + it only puts back what the rewrite would otherwise have destroyed.""" + if not snapshot or has_embedded_art(audio_file, symbols): + return False + restored = False + for kind, payload in snapshot: + try: + if kind == "flac" and hasattr(audio_file, "add_picture"): + for pic in payload: + audio_file.add_picture(pic) + restored = True + elif kind == "id3": + tags = getattr(audio_file, "tags", None) + if tags is not None: + for frame in payload: + tags.add(frame) + restored = True + elif kind == "mp4": + audio_file["covr"] = payload + restored = True + except Exception as exc: + logger.debug("restore_embedded_art (%s) failed: %s", kind, exc) + if restored: + logger.info("Preserved existing embedded cover art (re-embed produced none).") + return restored + + +__all__ = ["has_embedded_art", "snapshot_embedded_art", "restore_embedded_art", "ArtSnapshot"] diff --git a/core/metadata/artwork.py b/core/metadata/artwork.py index 9b8386af..a3c38c98 100644 --- a/core/metadata/artwork.py +++ b/core/metadata/artwork.py @@ -363,7 +363,7 @@ def embed_album_art_metadata(audio_file, metadata: dict): cfg = get_config_manager() symbols = get_mutagen_symbols() if not symbols: - return + return False try: image_data = None @@ -410,12 +410,12 @@ def embed_album_art_metadata(audio_file, metadata: dict): art_url = metadata.get("album_art_url") if not art_url: logger.warning("No album art URL available for embedding.") - return + return False image_data, mime_type = _fetch_art_bytes(art_url) if not image_data: logger.error("Failed to download album art data.") - return + return False if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.APIC(encoding=3, mime=mime_type, type=3, desc="Cover", data=image_data)) @@ -434,8 +434,10 @@ def embed_album_art_metadata(audio_file, metadata: dict): audio_file["covr"] = [symbols.MP4Cover(image_data, imageformat=fmt)] logger.info("Album art successfully embedded.") + return True except Exception as exc: logger.error("Error embedding album art: %s", exc) + return False def download_cover_art(album_info: dict, target_dir: str, context: dict = None): diff --git a/core/metadata/enrichment.py b/core/metadata/enrichment.py index 43e668e8..fdd34042 100644 --- a/core/metadata/enrichment.py +++ b/core/metadata/enrichment.py @@ -6,6 +6,10 @@ import os from types import SimpleNamespace from typing import Any +from core.metadata.art_preservation import ( + restore_embedded_art, + snapshot_embedded_art, +) from core.metadata.artwork import embed_album_art_metadata from core.metadata.common import ( get_config_manager, @@ -76,6 +80,8 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf file_lock = get_file_lock(file_path) with file_lock: logger.info("Enhancing metadata for: %s", os.path.basename(file_path)) + art_snapshot = [] # defined up front so the except handler can restore + audio_file = None try: strip_all_non_audio_tags(file_path) audio_file = symbols.File(file_path) @@ -83,6 +89,13 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf logger.error("Could not load audio file with Mutagen: %s", file_path) return False + # Capture any embedded cover art BEFORE we clear it. The rewrite + # below clears pictures/tags up front, but new art isn't fetched + # until much later (and may fail / be unavailable / be disabled). + # Without this snapshot, every such failure saves the file with + # the art destroyed and nothing put back — issue #764. + art_snapshot = snapshot_embedded_art(audio_file, symbols) + if hasattr(audio_file, "clear_pictures"): audio_file.clear_pictures() @@ -99,6 +112,9 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf metadata = extract_source_metadata(context, artist, album_info) if not metadata: logger.error("Could not extract source metadata, saving with cleared tags.") + # Don't destroy the original art just because we couldn't + # enrich the tags — put it back before saving. + restore_embedded_art(audio_file, symbols, art_snapshot) save_audio_file(audio_file, symbols) return True @@ -202,6 +218,13 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf elif isinstance(audio_file, symbols.MP4): audio_file["----:com.apple.iTunes:QUALITY"] = [symbols.MP4FreeForm(quality.encode("utf-8"))] + # If art embedding was skipped/disabled or produced nothing (no + # URL, download failed, rejected by the min-resolution guard), + # the file still has the pictures we cleared above. Restore the + # original so import never strips existing art (#764). No-op when + # new art was embedded — that path is unchanged. + restore_embedded_art(audio_file, symbols, art_snapshot) + save_audio_file(audio_file, symbols) verified = verify_metadata_written(file_path) @@ -219,4 +242,18 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf logger.warning("[Metadata Debug] Artist: %s", artist.get("name", "MISSING") if artist else "None") logger.warning("[Metadata Debug] Album info: %s", album_info.get("album_name", "MISSING") if album_info else "None") logger.error("[Metadata Debug] Traceback:\n%s", traceback.format_exc()) + # We cleared the file's art early; if the rewrite then crashed + # before re-embedding, the on-disk file (already saved cleared at + # the start) would be left art-less. Best-effort: put the original + # art back and persist it so a mid-enrichment crash never destroys + # the cover (#764). Guarded so a failure here can't mask the + # original error. + try: + if audio_file is not None and art_snapshot and restore_embedded_art( + audio_file, symbols, art_snapshot + ): + save_audio_file(audio_file, symbols) + logger.info("Restored original cover art after enrichment error.") + except Exception as restore_exc: + logger.debug("Art restore after error failed: %s", restore_exc) return False diff --git a/tests/test_art_preservation.py b/tests/test_art_preservation.py new file mode 100644 index 00000000..230f1420 --- /dev/null +++ b/tests/test_art_preservation.py @@ -0,0 +1,179 @@ +"""Tests for embedded-cover-art preservation across the enrichment rewrite. + +Regression for #764 (continuation of #755): importing a file destroyed its +embedded album art whenever the re-embed step couldn't produce new art +(no source metadata, no art URL, download failed, rejected by the min-size +guard, or art embedding disabled). ``enhance_file_metadata`` clears pictures +up front and saves regardless; these helpers snapshot the art first and put +it back iff the file would otherwise be saved with none. + +Uses real mutagen objects (a minimal valid FLAC + a minimal MP3) so the +snapshot/restore round-trips through the actual Picture/APIC APIs the +enricher uses — not a mock of them. +""" + +from __future__ import annotations + +import os +import tempfile + +import pytest + +mutagen = pytest.importorskip("mutagen") + +from mutagen.flac import FLAC, Picture # noqa: E402 +from mutagen.id3 import APIC, ID3, TIT2 # noqa: E402 + +from core.metadata.common import get_mutagen_symbols # noqa: E402 +from core.metadata.art_preservation import ( # noqa: E402 + has_embedded_art, + restore_embedded_art, + snapshot_embedded_art, +) + +SYMBOLS = get_mutagen_symbols() + +# 1x1 PNG — smallest valid image bytes for a real Picture/APIC payload. +_PNG = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01" + b"\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\x00" + b"\x01\x00\x00\x05\x00\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82" +) + + +def _minimal_flac_bytes() -> bytes: + # 4-byte magic + last STREAMINFO block (34 bytes). Mirrors the fixture + # used by tests/test_tag_writer_multi_artist.py. + return ( + b"fLaC" + + b"\x80\x00\x00\x22" + + b"\x00\x10\x00\x10" + + b"\x00\x00\x00\x00\x00\x00" + + b"\x0a\xc4\x42\xf0\x00\x00\x00\x00" + + b"\x00" * 16 + ) + + +@pytest.fixture +def flac_with_art(): + fd, path = tempfile.mkstemp(suffix=".flac") + os.close(fd) + with open(path, "wb") as f: + f.write(_minimal_flac_bytes()) + audio = FLAC(path) + pic = Picture() + pic.data = _PNG + pic.type = 3 + pic.mime = "image/png" + pic.width = 1 + pic.height = 1 + pic.depth = 24 + audio.add_picture(pic) + audio.save() + yield path + try: + os.remove(path) + except OSError: + pass + + +class _ID3Holder: + """Stand-in for an mutagen MP3 object: exposes a real ``ID3`` tag block + (the only thing the snapshot/restore helpers touch) without needing a + syncable MPEG frame on disk. ``__setitem__`` mirrors mutagen's mapping + sugar used by the MP4 restore branch — unused here but kept faithful.""" + + def __init__(self): + self.tags = ID3() + self.tags.add(TIT2(encoding=3, text=["original"])) + self.tags.add(APIC(encoding=3, mime="image/png", type=3, desc="Cover", data=_PNG)) + + def __setitem__(self, key, value): + self.tags[key] = value + + +@pytest.fixture +def mp3_with_art(): + return _ID3Holder() + + +# ── FLAC ──────────────────────────────────────────────────────────────── + + +def test_flac_art_restored_after_clear(flac_with_art): + audio = FLAC(flac_with_art) + assert has_embedded_art(audio, SYMBOLS) + + snap = snapshot_embedded_art(audio, SYMBOLS) + assert snap # something captured + + # Simulate the enrichment rewrite: nuke the art, fail to re-embed. + audio.clear_pictures() + assert not has_embedded_art(audio, SYMBOLS) + + restored = restore_embedded_art(audio, SYMBOLS, snap) + assert restored is True + assert has_embedded_art(audio, SYMBOLS) + assert audio.pictures[0].data == _PNG + + +def test_flac_restore_is_noop_when_new_art_present(flac_with_art): + # Happy path: re-embed succeeded, so restore must NOT touch the file. + audio = FLAC(flac_with_art) + snap = snapshot_embedded_art(audio, SYMBOLS) + + audio.clear_pictures() + new = Picture() + new.data = _PNG + b"NEWART" + new.type = 3 + new.mime = "image/png" + audio.add_picture(new) + + restored = restore_embedded_art(audio, SYMBOLS, snap) + assert restored is False + assert len(audio.pictures) == 1 + assert audio.pictures[0].data == _PNG + b"NEWART" # not clobbered/duplicated + + +def test_flac_no_art_snapshot_empty(): + fd, path = tempfile.mkstemp(suffix=".flac") + os.close(fd) + try: + with open(path, "wb") as f: + f.write(_minimal_flac_bytes()) + audio = FLAC(path) + assert snapshot_embedded_art(audio, SYMBOLS) == [] + # Restoring an empty snapshot is a no-op. + assert restore_embedded_art(audio, SYMBOLS, []) is False + finally: + os.remove(path) + + +# ── MP3 / ID3 ───────────────────────────────────────────────────────────── + + +def test_mp3_apic_restored_after_tags_cleared(mp3_with_art): + audio = mp3_with_art + assert has_embedded_art(audio, SYMBOLS) + snap = snapshot_embedded_art(audio, SYMBOLS) + assert snap + + # The enricher does audio_file.tags.clear() then rewrites tags. + audio.tags.clear() + assert not has_embedded_art(audio, SYMBOLS) + + restored = restore_embedded_art(audio, SYMBOLS, snap) + assert restored is True + apics = audio.tags.getall("APIC") + assert apics and apics[0].data == _PNG + + +def test_mp3_restore_noop_when_new_apic_present(mp3_with_art): + audio = mp3_with_art + snap = snapshot_embedded_art(audio, SYMBOLS) + audio.tags.clear() + audio.tags.add(APIC(encoding=3, mime="image/png", type=3, desc="Cover", data=_PNG + b"NEW")) + + assert restore_embedded_art(audio, SYMBOLS, snap) is False + apics = audio.tags.getall("APIC") + assert len(apics) == 1 and apics[0].data == _PNG + b"NEW" diff --git a/tests/test_enrichment_art_preservation.py b/tests/test_enrichment_art_preservation.py new file mode 100644 index 00000000..82f55def --- /dev/null +++ b/tests/test_enrichment_art_preservation.py @@ -0,0 +1,153 @@ +"""End-to-end proof that enhance_file_metadata never destroys embedded art. + +Bug #764: the enrichment rewrite clears cover art up front and saves the file +regardless of whether new art gets re-embedded. These tests run the REAL +``enhance_file_metadata`` against a REAL FLAC that has embedded art, and assert +the art is still on disk after every failure path — and that the happy path +(new art embedded) correctly REPLACES it. This exercises the actual wiring +(snapshot -> clear -> rewrite -> restore/save), not just the helper functions. + +Only the external collaborators are stubbed (config, source-metadata extraction, +the art fetch, source-id embed, verification) — the clear/snapshot/restore/save +sequence under test runs for real through mutagen. +""" + +from __future__ import annotations + +import os +import tempfile +from unittest.mock import patch + +import pytest + +pytest.importorskip("mutagen") +from mutagen.flac import FLAC, Picture # noqa: E402 + +import core.metadata.enrichment as enrichment # noqa: E402 + +_PNG = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01" + b"\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\x00" + b"\x01\x00\x00\x05\x00\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82" +) + + +class _Cfg: + """Config stub: returns the caller's default for every key, so + metadata_enhancement.enabled / embed_album_art resolve True.""" + + def get(self, key, default=None): + return default + + +def _make_flac_with_art(path): + minimal = ( + b"fLaC" + + b"\x80\x00\x00\x22" + + b"\x00\x10\x00\x10" + + b"\x00\x00\x00\x00\x00\x00" + + b"\x0a\xc4\x42\xf0\x00\x00\x00\x00" + + b"\x00" * 16 + ) + with open(path, "wb") as f: + f.write(minimal) + audio = FLAC(path) + pic = Picture() + pic.data = _PNG + pic.type = 3 + pic.mime = "image/png" + pic.width = 1 + pic.height = 1 + pic.depth = 24 + audio.add_picture(pic) + audio.save() + + +@pytest.fixture +def flac_path(): + fd, path = tempfile.mkstemp(suffix=".flac") + os.close(fd) + _make_flac_with_art(path) + yield path + try: + os.remove(path) + except OSError: + pass + + +def _disk_art(path): + """Return the embedded picture bytes on disk, or None.""" + pics = FLAC(path).pictures + return pics[0].data if pics else None + + +def _run(flac_path, *, metadata, embed_side_effect): + """Drive enhance_file_metadata with stubbed collaborators.""" + with patch.object(enrichment, "get_config_manager", return_value=_Cfg()), \ + patch.object(enrichment, "strip_all_non_audio_tags"), \ + patch.object(enrichment, "extract_source_metadata", return_value=metadata), \ + patch.object(enrichment, "embed_source_ids"), \ + patch.object(enrichment, "verify_metadata_written", return_value=True), \ + patch.object(enrichment, "embed_album_art_metadata", side_effect=embed_side_effect): + return enrichment.enhance_file_metadata( + flac_path, context={}, artist={"name": "Coldplay"}, album_info={}, + ) + + +# ── failure paths: art MUST survive ────────────────────────────────────── + + +def test_art_survives_when_source_metadata_missing(flac_path): + # extract_source_metadata returns None -> early return path. + assert _disk_art(flac_path) == _PNG + result = _run(flac_path, metadata=None, embed_side_effect=lambda *a, **k: False) + assert result is True + assert _disk_art(flac_path) == _PNG # art preserved on disk + + +def test_art_survives_when_embed_produces_nothing(flac_path): + # Metadata is fine, but the art fetch fails -> embed is a no-op (returns + # False, adds no picture). The original art must remain. + def embed_noop(audio_file, metadata): + return False # mirrors "no art URL" / "download failed" + + result = _run(flac_path, metadata={"title": "Yellow", "artist": "Coldplay"}, + embed_side_effect=embed_noop) + assert result is True + assert _disk_art(flac_path) == _PNG + + +def test_art_survives_when_embed_raises(flac_path): + # A hard crash mid-enrichment must not leave the file art-less. + def embed_boom(audio_file, metadata): + raise RuntimeError("art backend exploded") + + result = _run(flac_path, metadata={"title": "Yellow", "artist": "Coldplay"}, + embed_side_effect=embed_boom) + assert result is False # enrichment reported failure + assert _disk_art(flac_path) == _PNG # ...but art was restored on disk + + +# ── happy path: new art REPLACES old, no duplication ────────────────────── + + +def test_new_art_replaces_old_when_embed_succeeds(flac_path): + new_bytes = _PNG + b"BRANDNEW" + + def embed_real(audio_file, metadata): + # Simulate a successful fetch+embed: add the new picture in-place. + pic = Picture() + pic.data = new_bytes + pic.type = 3 + pic.mime = "image/png" + audio_file.add_picture(pic) + return True + + result = _run(flac_path, metadata={"title": "Yellow", "artist": "Coldplay"}, + embed_side_effect=embed_real) + assert result is True + on_disk = FLAC(flac_path).pictures + # Exactly one picture, and it's the NEW art — restore must not have + # re-added the old one on top. + assert len(on_disk) == 1 + assert on_disk[0].data == new_bytes From 3c15041b885d61cc5a82f5303a91c08ea9051d9e Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 08:40:26 -0700 Subject: [PATCH 003/108] Fix #764: manual import reported quarantined files as a successful "Done" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manual-import routes (album + singles) call post_process_matched_download directly. When the pipeline quarantines a file — integrity / AcoustID / FLAC bit-depth — or hits the race guard, it sets a context flag and RETURNS NORMALLY (it only marks the task failed + notifies when there's a task_id, which manual imports don't have). So the inner pipeline raised no exception, and routes.py counted `processed += 1` for a file that had just been moved to ss_quarantine, not the library. Result: the UI shows a green "Done" while the track silently vanished — exactly the #764 report (Coldplay - Yellow.flac -> ss_quarantine, but "Done"). The download path already handles this in post_process_matched_download_with_verification (it reads the same flags and marks the task failed); only the manual-import routes were missing the check. Fix: new pure helper import_rejection_reason(context) returns a human-readable reason for any terminal rejection (_integrity_failure_msg / _acoustid_quarantined / _bitdepth_rejected / _race_guard_failed) or None for a clean import. Both manual-import routes now consult it: album_process reports the track in `errors` instead of counting it processed; process_single_import_file returns ("error", reason) instead of ("ok", ...). Verified every move_to_quarantine call site (4, all in pipeline.py) sets one of those flags, so no quarantine path slips through. This also delivers the "direct display of the error" the reporter asked for — the reason now surfaces in the response `errors` list. Does NOT address the reverse symptom ("failed even though it moved correctly") — not yet root-caused — nor the separate bit-depth hole on the download-path wrapper. Tests: tests/imports/test_import_rejection_reason.py (10) — each trigger detected, falsy flags ignored, deterministic ordering, plus two route-level tests driving the REAL process_single_import_file (quarantine -> "error"; clean -> "ok"). --- core/imports/pipeline.py | 27 ++++ core/imports/routes.py | 21 ++- tests/imports/test_import_rejection_reason.py | 145 ++++++++++++++++++ 3 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 tests/imports/test_import_rejection_reason.py diff --git a/core/imports/pipeline.py b/core/imports/pipeline.py index fa196aa5..4263ba5a 100644 --- a/core/imports/pipeline.py +++ b/core/imports/pipeline.py @@ -110,6 +110,33 @@ def _mark_task_quarantined(context: dict, quarantine_path: str | None) -> None: download_tasks[task_id]['quarantine_entry_id'] = entry_id +def import_rejection_reason(context: dict) -> str | None: + """Human-readable reason if post-processing terminally rejected the file + (quarantine or race-guard), else ``None`` for a clean import. + + ``post_process_matched_download`` signals these outcomes by setting context + flags and returning normally — it only raises on unexpected errors. The + download path reads those flags in + ``post_process_matched_download_with_verification`` and marks the task + failed, but the MANUAL-import routes call ``post_process_matched_download`` + directly with no task_id, so without this check a quarantined file (now in + ss_quarantine, not the library) is counted as a successful import and the + UI shows a green "Done" (#764). Pure + testable: it only inspects the + context dict the inner pipeline populated.""" + if context.get('_integrity_failure_msg'): + return f"integrity check failed: {context['_integrity_failure_msg']}" + if context.get('_acoustid_quarantined'): + return ( + "AcoustID verification failed: " + f"{context.get('_acoustid_failure_msg', 'fingerprint mismatch')}" + ) + if context.get('_bitdepth_rejected'): + return "rejected by bit-depth filter" + if context.get('_race_guard_failed'): + return "source file disappeared before import completed" + return None + + def build_import_pipeline_runtime( *, automation_engine: Any | None = None, diff --git a/core/imports/routes.py b/core/imports/routes.py index edd38f08..1be49b49 100644 --- a/core/imports/routes.py +++ b/core/imports/routes.py @@ -11,6 +11,7 @@ from typing import Any, Callable, Dict from core.imports.album import build_album_import_context, build_album_import_match_payload, resolve_album_artist_context from core.imports.context import get_import_context_artist, get_import_track_info, normalize_import_context from core.imports.filename import parse_filename_metadata +from core.imports.pipeline import import_rejection_reason from core.imports.staging import ( AUDIO_EXTENSIONS, get_import_suggestions_cache, @@ -331,8 +332,17 @@ def album_process(runtime: ImportRouteRuntime, data: Dict[str, Any]) -> tuple[Di try: runtime.post_process_matched_download(context_key, context, file_path) - processed += 1 - runtime.logger.info("Import processed: %s. %s from %s", track_number, track_name, album_name) + # A quarantine/race-guard rejection returns normally (no + # exception) and leaves the file in ss_quarantine, NOT the + # library — so it must be reported as an error, not counted + # as a successful import (#764). + reject_reason = import_rejection_reason(context) + if reject_reason: + errors.append(f"{track_name}: {reject_reason}") + runtime.logger.warning("Import rejected: %s — %s", track_name, reject_reason) + else: + processed += 1 + runtime.logger.info("Import processed: %s. %s from %s", track_number, track_name, album_name) except Exception as proc_err: err_msg = f"{track_name}: {str(proc_err)}" errors.append(err_msg) @@ -422,6 +432,13 @@ def process_single_import_file(runtime: ImportRouteRuntime, file_info: Dict[str, context_key = f"import_single_{uuid.uuid4().hex[:8]}" runtime.post_process_matched_download(context_key, context, file_path) + # Quarantine/race-guard returns normally but the file is in + # ss_quarantine, not the library — report it as an error rather than + # "ok", else the UI shows a green "Done" for a file that vanished (#764). + reject_reason = import_rejection_reason(context) + if reject_reason: + runtime.logger.warning("Import single rejected: %s — %s", final_title, reject_reason) + return ("error", f"{final_title}: {reject_reason}") runtime.logger.info( "Import single processed: %s by %s (source=%s)", final_title, diff --git a/tests/imports/test_import_rejection_reason.py b/tests/imports/test_import_rejection_reason.py new file mode 100644 index 00000000..9377ff6a --- /dev/null +++ b/tests/imports/test_import_rejection_reason.py @@ -0,0 +1,145 @@ +"""Tests for import_rejection_reason — the manual-import quarantine guard. + +Regression for #764: the manual-import routes call +``post_process_matched_download`` directly. A quarantine (integrity / AcoustID +/ bit-depth) or race-guard rejection returns NORMALLY (no exception) and leaves +the file in ss_quarantine, not the library — but the routes counted any +no-exception return as a successful import, so the UI showed a green "Done" for +a file that had actually vanished. ``import_rejection_reason`` reads the context +flags the inner pipeline sets so the routes can report those as errors. +""" + +from __future__ import annotations + +import os +import tempfile + +from core.imports.pipeline import import_rejection_reason +from core.imports.routes import ImportRouteRuntime, process_single_import_file + + +def test_clean_import_returns_none(): + assert import_rejection_reason({}) is None + assert import_rejection_reason({'is_album': True, 'track_info': {}}) is None + + +def test_integrity_failure_detected(): + reason = import_rejection_reason({'_integrity_failure_msg': 'duration drift 12s'}) + assert reason is not None + assert 'integrity' in reason.lower() + assert 'duration drift 12s' in reason + + +def test_acoustid_quarantine_detected(): + reason = import_rejection_reason({ + '_acoustid_quarantined': True, + '_acoustid_failure_msg': 'wrong artist: got Oasis expected Coldplay', + }) + assert reason is not None + assert 'acoustid' in reason.lower() + assert 'Coldplay' in reason + + +def test_acoustid_quarantine_without_message_still_flags(): + # The flag alone must trip it even if no message was stashed. + reason = import_rejection_reason({'_acoustid_quarantined': True}) + assert reason is not None + assert 'acoustid' in reason.lower() + + +def test_bitdepth_rejection_detected(): + reason = import_rejection_reason({'_bitdepth_rejected': True}) + assert reason is not None + assert 'bit-depth' in reason.lower() + + +def test_race_guard_failure_detected(): + reason = import_rejection_reason({'_race_guard_failed': True}) + assert reason is not None + assert 'disappeared' in reason.lower() + + +def test_falsy_flags_do_not_trip(): + # A flag present but falsy (e.g. integrity passed) must NOT be a rejection. + ctx = { + '_integrity_failure_msg': '', + '_acoustid_quarantined': False, + '_bitdepth_rejected': False, + '_race_guard_failed': False, + } + assert import_rejection_reason(ctx) is None + + +def test_integrity_takes_precedence_when_multiple_set(): + # Deterministic ordering: integrity first. + reason = import_rejection_reason({ + '_integrity_failure_msg': 'truncated', + '_acoustid_quarantined': True, + '_bitdepth_rejected': True, + }) + assert 'integrity' in reason.lower() + + +# ── route-level wiring: a quarantine must NOT report as a successful import ── + + +def _runtime_with_post_process(post_process): + """Build an ImportRouteRuntime wired with stub resolvers + the supplied + post_process_matched_download. Resolvers return a shared context dict so a + flag the post-processor sets is what import_rejection_reason later reads.""" + ctx = {} + + return ImportRouteRuntime( + get_single_track_import_context=lambda *a, **k: {"context": ctx, "source": "local"}, + normalize_import_context=lambda c: c, + get_import_context_artist=lambda c: {"name": "Coldplay"}, + get_import_track_info=lambda c: {"name": "Yellow"}, + post_process_matched_download=post_process, + ), ctx + + +def _tmp_audio_file(): + fd, path = tempfile.mkstemp(suffix=".flac") + os.close(fd) + with open(path, "wb") as f: + f.write(b"fLaC") # only needs to exist for os.path.isfile + return path + + +def test_single_import_quarantine_reported_as_error(): + # post-processing quarantines the file (sets the flag, returns normally). + def quarantining_post_process(context_key, context, file_path): + context['_acoustid_quarantined'] = True + context['_acoustid_failure_msg'] = 'wrong track' + + runtime, _ctx = _runtime_with_post_process(quarantining_post_process) + path = _tmp_audio_file() + try: + outcome, payload = process_single_import_file( + runtime, {"full_path": path, "filename": "Coldplay - Yellow.flac", + "title": "Yellow", "artist": "Coldplay"}, + ) + finally: + os.remove(path) + + assert outcome == "error" # NOT "ok" -> route won't count it processed + assert "AcoustID" in payload + + +def test_single_import_clean_reports_ok(): + # post-processing succeeds (no flags) -> import counts as processed. + def clean_post_process(context_key, context, file_path): + return None + + runtime, _ctx = _runtime_with_post_process(clean_post_process) + path = _tmp_audio_file() + try: + outcome, payload = process_single_import_file( + runtime, {"full_path": path, "filename": "Coldplay - Yellow.flac", + "title": "Yellow", "artist": "Coldplay"}, + ) + finally: + os.remove(path) + + assert outcome == "ok" + assert payload == "Yellow" From 174513d3517fbb5f8a6f8d45576dd899fe73474b Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 09:14:26 -0700 Subject: [PATCH 004/108] Fix #769: playlist sync matched wrong same-artist track with high confidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tracks NOT in the library were matched to a DIFFERENT song by the SAME artist and reported with high confidence instead of as missing — e.g. "Dani California" -> "Californication" (Red Hot Chili Peppers), "Under The Bridge" -> "Around the World". Root cause: _calculate_track_confidence scores 0.5*title + 0.5*artist. A same-artist comparison always yields artist = 1.0, so the title score is the only thing that can tell two of an artist's songs apart — but that score is a SequenceMatcher CHARACTER ratio, which over-credits unrelated titles that share a long substring ("californi…" = 0.67) or just a stopword ("the" = 0.62). With the flat 0.5 artist term, anything clearing the weak 0.6 char floor lands at ~0.81-0.83, well over the 0.7 sync threshold. Reproduced on dev: both reported pairs score 0.81/0.83. Fix: new core/text/title_match.py:titles_plausibly_same, called in _calculate_track_confidence right before the floor. It accepts a pair only when it's near-identical char-wise (>=0.85, so typos / punctuation / casing like "Beleive"->"Believe", "HUMBLE."->"Humble" still match) OR the titles share at least one significant (non-stopword) word. Two different songs by the same artist share no content word, so they're rejected and the real track is correctly reported missing. ("the" is a stopword — that's what leaked "Under The Bridge"/"Around the World".) Scoped deliberately: the word-overlap test fires ONLY when at least one side has 2+ content words. For single-word titles there is no other word to share, so it defers to the existing char floor — otherwise legitimate stylized spellings ("Grey"/"Gray", "Tonite"/"Tonight", "4ever"/"Forever") would become new false-negatives. Verified those still match. The few single-word variants that do score low (Ok/Okay, Thru/Through) were already rejected by the pre-existing length-ratio penalty, not by this gate. Both reported false positives now score 0.33/0.31 -> missing. Does NOT address the harder case of two different same-artist songs that DO share a content word (e.g. "Believe"/"Believer") — pre-existing and unworsened. Any residual error fails safe: a false-missing is re-downloaded/wishlisted, vs the old behavior which silently substituted the wrong song. Tests: tests/test_title_match_guard.py (14) — pure-guard unit tests + a 13-pair battery driving the REAL _calculate_track_confidence (genuine matches stay >=0.7, same-artist different songs drop below), plus an explicit no-regression test for stylized single-word spellings. 292 matching/sync tests pass. --- core/text/title_match.py | 81 +++++++++++++++++ database/music_database.py | 16 ++++ tests/test_title_match_guard.py | 153 ++++++++++++++++++++++++++++++++ 3 files changed, 250 insertions(+) create mode 100644 core/text/title_match.py create mode 100644 tests/test_title_match_guard.py diff --git a/core/text/title_match.py b/core/text/title_match.py new file mode 100644 index 00000000..ed8ca4a2 --- /dev/null +++ b/core/text/title_match.py @@ -0,0 +1,81 @@ +"""Guard against char-level title false positives in track matching. + +Issue #769: playlist sync matched tracks that aren't in the library to a +DIFFERENT song by the SAME artist, with high confidence — e.g. "Dani +California" -> "Californication" (Red Hot Chili Peppers), "Under The Bridge" +-> "Around the World". The confidence formula is ``0.5*title + 0.5*artist``, +and a same-artist comparison always yields ``artist = 1.0``, so the title score +is the only thing that can tell two of an artist's songs apart. But the title +score is a ``difflib.SequenceMatcher`` character ratio, which over-credits +unrelated titles that happen to share a long substring ("californi…") or only a +stopword ("the"): 0.67 and 0.62 respectively. With the flat 0.5 artist term +that lands at 0.83 / 0.81 — well over the 0.7 sync threshold. + +``titles_plausibly_same`` adds a cheap word-level sanity check on top of the +char ratio: accept a pair only when it's near-identical char-wise (so typos and +punctuation/casing variants — "Beleive"/"Believe", "HUMBLE."/"Humble" — still +match) OR the two titles share at least one significant (non-stopword) token. +Two genuinely different songs by the same artist share no content word, so they +get rejected; the real track is then correctly reported missing. +""" + +from __future__ import annotations + +import re + +# Articles / prepositions / conjunctions only. Deliberately NOT pronouns +# ("you", "me", "i") — those carry meaning in song titles and dropping them +# could strip the only shared word from a real match. "the" MUST stay here: +# without it "Under The Bridge" and "Around the World" would falsely share it. +_TITLE_STOPWORDS = frozenset({ + "the", "a", "an", "of", "and", "or", "to", "in", "on", + "for", "with", "at", "by", "from", +}) + +_TOKEN_RE = re.compile(r"[a-z0-9]+") + +# Char ratio at/above which two titles are treated as the same regardless of +# shared words — covers typos, punctuation, casing, accents. Tuned so single- +# word typos ("Beleive"/"Believe" = 0.857) pass while the #769 false positives +# ("Dani California"/"Californication" = 0.667) do not. +_NEAR_IDENTICAL = 0.85 + + +def _content_tokens(text: str) -> set[str]: + return {t for t in _TOKEN_RE.findall((text or "").lower()) if t not in _TITLE_STOPWORDS} + + +def titles_plausibly_same( + title_a: str, + title_b: str, + char_similarity: float, + *, + near_identical: float = _NEAR_IDENTICAL, +) -> bool: + """Whether two titles could be the same track, given their char similarity. + + ``title_a`` / ``title_b`` should already be normalised/cleaned (lowercased, + brackets stripped) the same way the caller computed ``char_similarity``. + + Returns ``True`` when the pair is near-identical char-wise OR shares at + least one significant (non-stopword) token. Returns ``False`` for two + titles that are only moderately char-similar and share no content word — + i.e. different songs the char ratio over-credited (#769).""" + if char_similarity >= near_identical: + return True + ta = _content_tokens(title_a) + tb = _content_tokens(title_b) + # Word-overlap is only a reliable "different song" signal when at least one + # side has 2+ content words — that's the #769 case where the char ratio + # over-credits a shared substring ("Dani California"/"Californication") or + # a stopword ("Under The Bridge"/"Around the World"). For single-word + # titles there's no other word to share, so applying it would wrongly fail + # legitimate stylized spellings ("Grey"/"Gray", "Tonite"/"Tonight", + # "Thru"/"Through") that the char ratio rightly accepts. In that case defer + # to the caller's existing char-similarity floor instead of force-failing. + if max(len(ta), len(tb)) < 2 or not ta or not tb: + return True + return not ta.isdisjoint(tb) + + +__all__ = ["titles_plausibly_same"] diff --git a/database/music_database.py b/database/music_database.py index 85beef86..08d42d14 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -7248,6 +7248,22 @@ class MusicDatabase: # Titles differ in length by more than 30% — penalize heavily best_title_similarity *= len_ratio + # Word-level guard: SequenceMatcher's char ratio over-credits + # different songs that share a long substring or only a stopword + # ("Dani California" vs "Californication" = 0.67; "Under The Bridge" + # vs "Around the World" = 0.62). Since a same-artist comparison + # always scores artist = 1.0, the title is the only discriminator, + # so a bad-but-moderate title score gets carried over the threshold + # (#769). Reject pairs that aren't near-identical AND share no + # significant word — the real track is then reported missing. + from core.text.title_match import titles_plausibly_same + if not titles_plausibly_same( + clean_search_title or search_title_norm, + clean_db_title or db_title_norm, + best_title_similarity, + ): + return best_title_similarity * 0.5 # below any threshold + # Require minimum title similarity to prevent a perfect artist match from # carrying a bad title match over the threshold (e.g. "Time" vs "Time Flies") if best_title_similarity < 0.6: diff --git a/tests/test_title_match_guard.py b/tests/test_title_match_guard.py new file mode 100644 index 00000000..49fb5a6a --- /dev/null +++ b/tests/test_title_match_guard.py @@ -0,0 +1,153 @@ +"""Tests for the title word-overlap guard (#769). + +Playlist sync matched tracks NOT in the library to a different song by the +SAME artist with high confidence ("Dani California" -> "Californication"; +"Under The Bridge" -> "Around the World"). Root cause: confidence is +0.5*title + 0.5*artist, same-artist always gives artist=1.0, and the title +score is a SequenceMatcher char ratio that over-credits unrelated titles +sharing a substring or a stopword. titles_plausibly_same gates those out. + +Two layers tested: + 1. titles_plausibly_same in isolation (the pure decision). + 2. the real _calculate_track_confidence end-to-end, asserting the two + reported false positives now fall below the 0.7 sync threshold while a + battery of genuine matches stays above it. +""" + +from __future__ import annotations + +import types + +from core.text.title_match import titles_plausibly_same + + +# ── the pure guard ──────────────────────────────────────────────────────── + + +def test_near_identical_passes_even_without_shared_token(): + # Single-word typo: no shared token, but char-identical enough. + assert titles_plausibly_same("beleive", "believe", 0.857) is True + + +def test_punctuation_casing_variants_pass(): + assert titles_plausibly_same("humble", "humble", 0.92) is True + + +def test_shared_significant_word_passes_below_near_identical(): + # Moderate char score but a real shared content word. + assert titles_plausibly_same("hello world", "hello there", 0.6) is True + + +def test_different_songs_sharing_only_substring_rejected(): + # #769: "Dani California" vs "Californication" — share the substring + # "californi" (high char ratio) but no whole word. + assert titles_plausibly_same("dani california", "californication", 0.667) is False + + +def test_different_songs_sharing_only_stopword_rejected(): + # #769: "Under The Bridge" vs "Around the World" — share only "the". + assert titles_plausibly_same("under the bridge", "around the world", 0.625) is False + + +def test_multiword_stopword_only_overlap_rejected(): + # Two 2+-word titles sharing only "the" — the #769 shape. + assert titles_plausibly_same("under the bridge", "around the world", 0.625) is False + + +def test_single_word_titles_defer_to_char_floor(): + # Single content word on each side: no "other word" to share, so the gate + # must NOT force-fail — it defers (returns True) and lets the caller's char + # floor decide. This is what protects stylized spellings like "Grey"/"Gray" + # and "Tonite"/"Tonight" from becoming new false negatives. + assert titles_plausibly_same("grey", "gray", 0.75) is True + assert titles_plausibly_same("tonite", "tonight", 0.77) is True + # ...even when the char score is low — the floor, not the gate, rejects it. + assert titles_plausibly_same("numb", "creep", 0.2) is True + + +def test_all_stopword_side_defers(): + # One side is all stopwords -> no word signal -> defer to char floor. + assert titles_plausibly_same("the the", "around the world", 0.5) is True + + +# ── end-to-end through the real confidence scorer ────────────────────────── + +from database.music_database import MusicDatabase # noqa: E402 + +_THRESHOLD = 0.7 # services/sync_service.py confidence_threshold + + +class _FakeTrack: + def __init__(self, title, artist): + self.title = title + self.artist_name = artist + self.track_artist = None + + +def _scorer(): + stub = type("S", (), {})() + for m in ( + "_calculate_track_confidence", "_string_similarity", + "_normalize_for_comparison", "_clean_track_title_for_comparison", + ): + setattr(stub, m, types.MethodType(getattr(MusicDatabase, m), stub)) + return stub + + +# (source_title, library_title, same_artist, should_match) +_BATTERY = [ + # genuine matches — must stay matched + ("Mr. Brightside", "Mr Brightside", True), + ("HUMBLE.", "Humble", True), + ("Beleive", "Believe", True), # typo + ("In the End", "In The End", True), + ("thank u, next", "Thank U Next", True), + ("Old Town Road", "Old Town Road (feat. Billy Ray Cyrus)", True), + ("bad guy", "bad guy", True), + # different songs by the SAME artist — must be reported missing + ("Dani California", "Californication", False), # the reported case + ("Under The Bridge", "Around the World", False), # the reported case + ("Otherside", "Californication", False), + ("Numb", "In the End", False), + ("Yellow", "The Scientist", False), + ("Seven Nation Army", "Fell in Love with a Girl", False), +] + + +def test_confidence_battery_separates_real_from_false_matches(): + s = _scorer() + artist = "Red Hot Chili Peppers" + misclassified = [] + for src, lib, should_match in _BATTERY: + conf = s._calculate_track_confidence(src, artist, _FakeTrack(lib, artist)) + matched = conf >= _THRESHOLD + if matched != should_match: + misclassified.append((src, lib, should_match, round(conf, 3))) + assert not misclassified, f"misclassified: {misclassified}" + + +def test_reported_false_positives_now_below_threshold(): + s = _scorer() + a = "Red Hot Chili Peppers" + assert s._calculate_track_confidence("Dani California", a, _FakeTrack("Californication", a)) < _THRESHOLD + assert s._calculate_track_confidence("Under The Bridge", a, _FakeTrack("Around the World", a)) < _THRESHOLD + + +def test_exact_title_same_artist_still_perfect(): + s = _scorer() + a = "Garbage" + conf = s._calculate_track_confidence("Only Happy When It Rains", a, + _FakeTrack("Only Happy When It Rains", a)) + assert conf >= 0.99 + + +def test_single_word_spelling_variants_not_regressed(): + # The gate must not turn legitimate stylized single-word spellings into + # new "missing" reports (the regression the first cut of this fix had). + # These all matched before #769's gate and must still match. + s = _scorer() + a = "Some Artist" + for src, lib in [("Grey", "Gray"), ("Tonite", "Tonight"), + ("4ever", "Forever"), ("Lovin'", "Loving"), ("Colour", "Color")]: + conf = s._calculate_track_confidence(src, a, _FakeTrack(lib, a)) + assert conf >= _THRESHOLD, f"{src!r}->{lib!r} regressed to {conf:.3f}" From bba083632411433c163ce061628f4af577b7cfc2 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 10:16:21 -0700 Subject: [PATCH 005/108] Fix #768: playlist sync editor refusing to match certain tracks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three compounding bugs hit tracks whose source metadata is YouTube/streaming- shaped — title "Artist - Song", artist "Official Artist"/"Artist - Topic"/ "ArtistVEVO" (reported: "Arctic Monkeys - Do I Wanna Know?" by "Official Arctic Monkeys"). Server-agnostic — affects Plex/Jellyfin/Navidrome, not just the reporter's Navidrome. Bug A — the match fails. The confidence scorer and the editor's reconcile both compared the raw "Artist - Song" title against the library's clean "Song"; the length-ratio penalty + floor drove it to ~0.18 (NO-MATCH), so the track showed unmatched while its server copy showed as an orphan "extra". New pure core/text/source_title.py (clean_source_artist / strip_artist_prefix / canonical_source_track) strips the channel/video decoration, applied at BOTH matching seams: services/sync_service._find_track_in_media_server (tries raw then canonical, keeps the best) and the editor reconcile. Conservative: a title prefix is stripped only when it equals the artist, so "Self-Titled", "Jay-Z", and "Marvin Gaye" (by another artist) are untouched, and the canonical form is an additional best-of candidate so it can only help. Bug B — manual matches never persisted. get_server_playlist_tracks built the per-source entry WITHOUT source_track_id, so "Find & add" posted an empty id and _persist_find_and_add_match returned early. The match reverted to "extra" on reload and re-adding looped. The editor's 3-pass matcher is now lifted to a pure, tested core.sync.playlist_reconcile.reconcile_playlist that includes source_track_id (the frontend at pages-extra.js:1836 already reads + sends it). Bug C — manual match duplicated + delete wiped all copies. "Find & add" always inserted, so linking a source to an already-present server track appended a duplicate (pos 72, 73...); remove filtered out EVERY entry with the target id. New pure core.sync.playlist_edit (plan_playlist_add: link-don't-duplicate when the target is already present; remove_one_occurrence: drop a single copy) wired into the Plex/Jellyfin/Navidrome add + remove branches. Tests (extreme): tests/test_source_title.py (35), tests/test_playlist_reconcile.py (11 — incl. the reported case, parity for override/exact/fuzzy/extra, and duplicate-server handling), tests/test_playlist_edit.py (12). 286 matching/sync tests still pass. Caveats: the sync_service change and the add/remove/editor endpoints are read-verified, not executed against a live media server (none in CI). The pure cores they call are exhaustively unit-tested; output-shape parity of the reconcile lift is covered. Delete removes the first matching copy (duplicates are identical, so harmless). --- core/sync/playlist_edit.py | 74 ++++++++++++ core/sync/playlist_reconcile.py | 186 +++++++++++++++++++++++++++++++ core/text/source_title.py | 105 +++++++++++++++++ services/sync_service.py | 34 ++++-- tests/test_playlist_edit.py | 80 +++++++++++++ tests/test_playlist_reconcile.py | 122 ++++++++++++++++++++ tests/test_source_title.py | 113 +++++++++++++++++++ web_server.py | 175 +++++++++-------------------- 8 files changed, 761 insertions(+), 128 deletions(-) create mode 100644 core/sync/playlist_edit.py create mode 100644 core/sync/playlist_reconcile.py create mode 100644 core/text/source_title.py create mode 100644 tests/test_playlist_edit.py create mode 100644 tests/test_playlist_reconcile.py create mode 100644 tests/test_source_title.py diff --git a/core/sync/playlist_edit.py b/core/sync/playlist_edit.py new file mode 100644 index 00000000..c2cf34d5 --- /dev/null +++ b/core/sync/playlist_edit.py @@ -0,0 +1,74 @@ +"""Pure planners for sync-editor playlist mutations (#768 Bug C). + +The sync editor's "Find & add" and remove actions rewrite the whole server +playlist from a flat list of track IDs (Subsonic/Navidrome + Jellyfin have no +position-level ops). Two bugs lived in the inline endpoint logic: + +* **Duplicate on manual match.** "Find & add" always *inserted* the chosen + track — but when the user is matching an UNMATCHED source to a server track + that's already in the playlist (an orphan "extra"), the intent is to LINK + them, not add a second copy. Each attempt appended another duplicate + (positions 72, 73, 74…). ``plan_playlist_add`` skips the insert when it's a + link to an already-present track (the caller still persists the override). + +* **Delete removes ALL copies.** The inline remove filtered out *every* entry + with the target ID. With duplicates present, deleting one removed them all. + ``remove_one_occurrence`` drops a single entry (duplicates are the same + track, so removing any one is correct). + +Pure, no I/O — the caller fetches the current track-id list and applies the +returned plan to the media-server client. +""" + +from __future__ import annotations + +from typing import List, Optional, Tuple + + +def plan_playlist_add( + current_ids: List[str], + track_id: str, + *, + is_link: bool, + position: Optional[int] = None, +) -> dict: + """Plan a "Find & add" against a flat track-id playlist. + + ``is_link`` is True when the add carries a ``source_track_id`` (i.e. the + user is matching an unmatched source to this server track). In that case, + if the track is ALREADY in the playlist, return ``should_insert=False`` so + the caller only records the override and never duplicates it. + + Returns ``{'should_insert': bool, 'new_ids': [...]}``. ``new_ids`` equals + the input (stringified) when no insert is needed.""" + tid = str(track_id) + current = [str(t) for t in current_ids] + if is_link and tid in current: + return {"should_insert": False, "new_ids": current} + pos = len(current) if position is None else max(0, min(int(position), len(current))) + new_ids = current[:pos] + [tid] + current[pos:] + return {"should_insert": True, "new_ids": new_ids} + + +def remove_one_occurrence( + track_ids: List[str], + target_id: str, + position: Optional[int] = None, +) -> Tuple[List[str], bool]: + """Remove a SINGLE occurrence of ``target_id`` from a flat id list. + + If ``position`` is given and the id there matches, that exact entry is + removed (so the user removes the row they clicked); otherwise the first + matching id is removed. Returns ``(new_ids, removed)``. ``removed`` is + False when the id isn't present (caller should 404).""" + target = str(target_id) + ids = [str(t) for t in track_ids] + if position is not None and 0 <= position < len(ids) and ids[position] == target: + return ids[:position] + ids[position + 1:], True + for idx, tid in enumerate(ids): + if tid == target: + return ids[:idx] + ids[idx + 1:], True + return ids, False + + +__all__ = ["plan_playlist_add", "remove_one_occurrence"] diff --git a/core/sync/playlist_reconcile.py b/core/sync/playlist_reconcile.py new file mode 100644 index 00000000..8a37fafc --- /dev/null +++ b/core/sync/playlist_reconcile.py @@ -0,0 +1,186 @@ +"""Reconcile a source playlist against a media-server playlist (pure). + +Lifted verbatim from the inline three-pass matcher in +``web_server.get_server_playlist_tracks`` so it can be unit-tested and so the +two #768 fixes live in importable, covered code: + + Pass 0 user-confirmed overrides (``sync_match_cache``), applied first. + Pass 1 exact normalized-title match. + Pass 2 fuzzy match on ``"artist title"`` (SequenceMatcher >= 0.75). + Extra server tracks no source claimed -> ``match_status='extra'``. + +Two bug fixes over the original inline version: + +* #768 Bug A — the source side is YouTube/streaming-shaped (title + ``"Artist - Song"``, artist ``"Official Artist"``). The original passes + compared the raw title, so ``"Arctic Monkeys - Do I Wanna Know?"`` never + matched the library's ``"Do I Wanna Know?"`` and the track showed as + unmatched while its server copy showed as an orphan "extra". We now also try + the canonicalized source title/artist (see ``core.text.source_title``). + +* #768 Bug B — the original built the per-source ``src_entry`` WITHOUT + ``source_track_id``, so the editor UI never received it; "Find & add" then + posted an empty id and the manual match was never persisted (it reverted to + "extra" on reload, and re-adding duplicated the track). ``src_entry`` now + carries ``source_track_id``. + +Pure, no I/O. ``override_pairs`` (``{source_idx: server_idx}``) is computed by +the caller via ``core.sync.match_overrides.resolve_match_overrides`` so the DB +lookup stays out of this module. +""" + +from __future__ import annotations + +import re +from difflib import SequenceMatcher +from typing import Any, Dict, List, Optional + +from core.text.source_title import canonical_source_track + +_FUZZY_THRESHOLD = 0.75 + +_FEAT_RE = re.compile(r'\s*[\(\[](?:feat|ft)\.?[^\)\]]*[\)\]]', re.IGNORECASE) +_REMASTER_RE = re.compile( + r'\s*[\(\[](?:\d{4}\s+)?remaster(?:ed)?(?:\s+version)?\s*[\)\]]', re.IGNORECASE) +_EDITION_RE = re.compile( + r'\s*[\(\[](?:deluxe|special|anniversary|legacy|expanded|limited)(?:\s+edition)?\s*[\)\]]', + re.IGNORECASE) + + +def norm_title(t: str) -> str: + """Strip feat./ft., remaster, and edition qualifiers for comparison only. + Byte-faithful to web_server's inline ``_norm_title``.""" + t = _FEAT_RE.sub('', t or '') + t = _REMASTER_RE.sub('', t) + t = _EDITION_RE.sub('', t) + return t.lower().strip() + + +def _src_entry(src: Dict[str, Any], position_fallback: int) -> Dict[str, Any]: + return { + 'name': src.get('name', ''), + 'artist': src.get('artist', ''), + 'album': src.get('album', ''), + 'image_url': src.get('image_url', ''), + 'duration_ms': src.get('duration_ms', 0), + 'position': src.get('position', position_fallback), + # #768 Bug B: echo the source id back so the editor can persist a + # manual "Find & add" override against it. + 'source_track_id': src.get('source_track_id', '') or '', + } + + +def _resolved_artist(src: Dict[str, Any]) -> str: + """Artist string, falling back to the first of an ``artists`` list.""" + artist = src.get('artist', '') + if not artist and src.get('artists'): + a = src['artists'][0] if src['artists'] else '' + artist = a.get('name', a) if isinstance(a, dict) else str(a) + return artist or '' + + +def reconcile_playlist( + source_tracks: List[Dict[str, Any]], + server_tracks: List[Dict[str, Any]], + override_pairs: Optional[Dict[int, int]] = None, +) -> List[Dict[str, Any]]: + """Return the combined matched/missing/extra view (list of dicts). + + Each combined entry has ``source_track``, ``server_track``, + ``match_status`` ('matched'|'missing'|'extra'), ``confidence``, and + ``override: True`` on override hits.""" + override_pairs = override_pairs or {} + combined: List[Dict[str, Any]] = [] + used_server_indices: set[int] = set() + unmatched_source: List[tuple[int, Dict[str, Any], str]] = [] + + # Precompute normalized server titles once. + server_norm = [norm_title(svr.get('title', '')) for svr in server_tracks] + + for i, src in enumerate(source_tracks): + src_artist = _resolved_artist(src) + src_name = src.get('name', '') + src_entry = _src_entry({**src, 'artist': src_artist}, i) + + # Pass 0: user-confirmed override. + if i in override_pairs: + j = override_pairs[i] + used_server_indices.add(j) + combined.append({ + 'source_track': src_entry, + 'server_track': server_tracks[j], + 'match_status': 'matched', + 'confidence': 1.0, + 'override': True, + }) + continue + + # Pass 1: exact normalized-title match — try the raw source title AND + # the canonicalized one (strips "Artist - " prefix / channel artist). + canon_title, _canon_artist = canonical_source_track(src_name, src_artist) + candidates = {norm_title(src_name), norm_title(canon_title)} + best_idx = -1 + for j, svr_norm in enumerate(server_norm): + if j in used_server_indices: + continue + if svr_norm in candidates: + best_idx = j + break + + if best_idx >= 0: + used_server_indices.add(best_idx) + combined.append({ + 'source_track': src_entry, + 'server_track': server_tracks[best_idx], + 'match_status': 'matched', + 'confidence': 1.0, + }) + else: + idx = len(combined) + combined.append({ + 'source_track': src_entry, + 'server_track': None, + 'match_status': 'missing', + 'confidence': 0.0, + }) + # Carry the canonical artist for the fuzzy pass. + unmatched_source.append((idx, src_entry, _canon_artist or src_artist)) + + # Pass 2: fuzzy match on remaining unmatched source tracks. Build the key + # from the canonicalized title/artist so YouTube-shaped sources can pair. + for combo_idx, src_entry, canon_artist in unmatched_source: + canon_title, _ = canonical_source_track(src_entry['name'], src_entry['artist']) + src_key = f"{canon_artist} {norm_title(canon_title)}".strip().lower() + best_score = 0.0 + best_j = -1 + for j, svr in enumerate(server_tracks): + if j in used_server_indices: + continue + svr_key = f"{svr.get('artist', '')} {norm_title(svr.get('title', ''))}".strip().lower() + score = SequenceMatcher(None, src_key, svr_key).ratio() + if score > best_score and score >= _FUZZY_THRESHOLD: + best_score = score + best_j = j + if best_j >= 0: + used_server_indices.add(best_j) + combined[combo_idx] = { + 'source_track': src_entry, + 'server_track': server_tracks[best_j], + 'match_status': 'matched', + 'confidence': round(best_score, 3), + } + + # Extra: server tracks no source claimed. + for j, svr in enumerate(server_tracks): + if j not in used_server_indices: + combined.append({ + 'source_track': None, + 'server_track': svr, + 'match_status': 'extra', + 'confidence': 0.0, + }) + + return combined + + +__all__ = ["reconcile_playlist", "norm_title"] diff --git a/core/text/source_title.py b/core/text/source_title.py new file mode 100644 index 00000000..03c80e2a --- /dev/null +++ b/core/text/source_title.py @@ -0,0 +1,105 @@ +"""Normalize streaming/YouTube-style source track metadata for matching. + +Issue #768: source playlists — especially ones seeded from YouTube — carry +video-style metadata: the title is ``"Artist - Song"`` and the artist is a +channel name like ``"Official Arctic Monkeys"``, ``"Arctic Monkeys - Topic"``, +or ``"ColdplayVEVO"``. The library/media-server side has the clean ``"Song"`` / +``"Arctic Monkeys"``. Both matching paths (the sync confidence scorer and the +playlist-editor reconcile) then fail to pair them — the track is reported +"not matched" / shows up as an orphan "extra" even though it exists. + +These helpers strip that channel/video decoration so the cleaned source can be +compared against the clean library metadata. Pure, no I/O. + +Conservative by construction: +- ``strip_artist_prefix`` removes a leading ``""`` only when the + prefix EQUALS the artist we're matching against. So ``"Death - Pull the + Plug"`` by ``"Death"`` is helped, while ``"Marvin Gaye"`` by Charlie Puth + (title is not ``"Charlie Puth - ..."``) is left untouched, and a hyphenated + word like ``"Self-Titled"`` is never split (a separator needs surrounding + whitespace, or a colon). +- ``clean_source_artist`` only removes well-known channel decorations. + +Both are intended to be applied as ADDITIONAL match candidates (best-of), so +an over-eager strip can only add a comparison, never remove the original. +""" + +from __future__ import annotations + +import re + +from core.text.normalize import normalize_for_comparison + +# Artist/title separator: a dash/pipe/tilde flanked by whitespace, OR a colon +# (with optional trailing space). Whitespace-flanking keeps "Self-Titled" and +# "Jay-Z" intact while still splitting "Artist - Title". +_SEP_SPLIT = re.compile(r"\s+[-–—|~]\s+|\s*:\s+") + +# YouTube auto-generated artist channel: "Arctic Monkeys - Topic". +_TOPIC_SUFFIX = re.compile(r"\s*-\s*topic\s*$", re.IGNORECASE) +# "Official " / "The Official " channel prefix. +_OFFICIAL_PREFIX = re.compile(r"^\s*(?:the\s+)?official\s+", re.IGNORECASE) +# Trailing VEVO, attached ("ColdplayVEVO") or spaced ("Coldplay VEVO"). +_VEVO_SUFFIX = re.compile(r"\s*vevo\s*$", re.IGNORECASE) + + +def clean_source_artist(artist: str) -> str: + """Strip well-known streaming-channel decoration from an artist name. + + ``"Official Arctic Monkeys"`` → ``"Arctic Monkeys"``; + ``"Arctic Monkeys - Topic"`` → ``"Arctic Monkeys"``; + ``"ColdplayVEVO"`` → ``"Coldplay"``. Returns the input unchanged when + nothing matches, and never returns empty for non-empty input.""" + if not artist: + return artist + s = artist.strip() + + topic = _TOPIC_SUFFIX.sub("", s).strip() + if topic and topic != s: + s = topic + + official = _OFFICIAL_PREFIX.sub("", s).strip() + if official: + s = official + + # Only strip VEVO if at least 2 chars of name remain (don't empty "VEVO"). + vevo = _VEVO_SUFFIX.sub("", s).strip() + if len(vevo) >= 2 and vevo != s: + s = vevo + + return s or artist + + +def strip_artist_prefix(title: str, artist: str) -> str: + """Remove a leading ``""`` from ``title`` when the prefix + equals ``artist`` (accent/case-folded). Otherwise return ``title`` unchanged. + + ``("Arctic Monkeys - Do I Wanna Know?", "Arctic Monkeys")`` → ``"Do I Wanna + Know?"``. Never returns an empty string.""" + if not title or not artist: + return title + na = normalize_for_comparison(artist) + if not na: + return title + parts = _SEP_SPLIT.split(title, maxsplit=1) + if len(parts) == 2: + left, right = parts + right = right.strip() + if right and normalize_for_comparison(left) == na: + return right + return title + + +def canonical_source_track(title: str, artist: str) -> tuple[str, str]: + """Best-effort clean (title, artist) for matching a streaming/YouTube source + against clean library metadata. Cleans the artist first, then strips a + leading artist prefix from the title using EITHER the cleaned or the raw + artist (YouTube titles prepend the real artist, not the channel name).""" + cleaned_artist = clean_source_artist(artist) + new_title = strip_artist_prefix(title, cleaned_artist) + if new_title == title and cleaned_artist != artist: + new_title = strip_artist_prefix(title, artist) + return new_title, cleaned_artist + + +__all__ = ["clean_source_artist", "strip_artist_prefix", "canonical_source_track"] diff --git a/services/sync_service.py b/services/sync_service.py index e51980ef..58ddfa18 100644 --- a/services/sync_service.py +++ b/services/sync_service.py @@ -586,17 +586,35 @@ class PlaylistSyncService: artist_name = _artist_name(artist) + # YouTube/streaming sources arrive video-shaped: title + # "Artist - Song", artist "Official Artist"/"Artist - Topic"/ + # "ArtistVEVO". Try the raw (title, artist) first, then a + # canonicalized variant so those still match the clean library + # metadata instead of being reported missing (#768). + from core.text.source_title import canonical_source_track + _attempts = [(original_title, artist_name)] + _canon_title, _canon_artist = canonical_source_track(original_title, artist_name) + if (_canon_title, _canon_artist) != (original_title, artist_name): + _attempts.append((_canon_title, _canon_artist)) + # Use the improved database check_track_exists method with server awareness try: db = MusicDatabase() - artist_candidates = self._get_or_fetch_artist_candidates( - candidate_pool, db, artist_name, active_server, - ) - db_track, confidence = db.check_track_exists( - original_title, artist_name, - confidence_threshold=0.7, server_source=active_server, - candidate_tracks=artist_candidates, - ) + # Try every candidate form and keep the BEST — don't stop at + # the first that clears the threshold, or a marginal raw + # match could mask a stronger (correct) canonical one. + db_track, confidence = None, 0.0 + for _try_title, _try_artist in _attempts: + artist_candidates = self._get_or_fetch_artist_candidates( + candidate_pool, db, _try_artist, active_server, + ) + _cand_track, _cand_conf = db.check_track_exists( + _try_title, _try_artist, + confidence_threshold=0.7, server_source=active_server, + candidate_tracks=artist_candidates, + ) + if _cand_conf > confidence: + db_track, confidence = _cand_track, _cand_conf if db_track and confidence >= 0.7: logger.debug(f"Database match found for '{original_title}' by '{artist_name}': '{db_track.title}' with confidence {confidence:.2f}") diff --git a/tests/test_playlist_edit.py b/tests/test_playlist_edit.py new file mode 100644 index 00000000..13a2f041 --- /dev/null +++ b/tests/test_playlist_edit.py @@ -0,0 +1,80 @@ +"""Extreme battery for sync-editor add/remove planners (#768 Bug C).""" + +from __future__ import annotations + +from core.sync.playlist_edit import plan_playlist_add, remove_one_occurrence + + +# ── plan_playlist_add: link must not duplicate ──────────────────────────── + +def test_link_to_existing_track_does_not_insert(): + # The reported loop: matching an unmatched source to a track already in + # the playlist (an "extra") must NOT add a second copy. + plan = plan_playlist_add(["a", "b", "nv72"], "nv72", is_link=True) + assert plan["should_insert"] is False + assert plan["new_ids"] == ["a", "b", "nv72"] # unchanged + + +def test_link_to_absent_track_inserts(): + plan = plan_playlist_add(["a", "b"], "nv99", is_link=True, position=1) + assert plan["should_insert"] is True + assert plan["new_ids"] == ["a", "nv99", "b"] + + +def test_non_link_add_always_inserts_even_if_present(): + # A plain add (no source link) may legitimately duplicate. + plan = plan_playlist_add(["a", "b"], "a", is_link=False) + assert plan["should_insert"] is True + assert plan["new_ids"].count("a") == 2 + + +def test_add_appends_when_no_position(): + plan = plan_playlist_add(["a", "b"], "c", is_link=False) + assert plan["new_ids"] == ["a", "b", "c"] + + +def test_add_clamps_out_of_range_position(): + assert plan_playlist_add(["a"], "c", is_link=False, position=99)["new_ids"] == ["a", "c"] + assert plan_playlist_add(["a"], "c", is_link=False, position=-5)["new_ids"] == ["c", "a"] + + +def test_add_stringifies_ids(): + plan = plan_playlist_add([1, 2, 72], 72, is_link=True) + assert plan["should_insert"] is False + + +# ── remove_one_occurrence: remove ONE, not all ──────────────────────────── + +def test_removes_only_one_of_duplicates(): + # The #768 delete bug: two copies (pos 72, 73) — removing must drop ONE. + new_ids, removed = remove_one_occurrence(["a", "nv72", "nv72", "b"], "nv72") + assert removed is True + assert new_ids == ["a", "nv72", "b"] # one copy survives + + +def test_removes_exact_position_when_given(): + new_ids, removed = remove_one_occurrence(["x", "x", "x"], "x", position=1) + assert removed is True + assert new_ids == ["x", "x"] + + +def test_falls_back_to_first_when_position_mismatches(): + new_ids, removed = remove_one_occurrence(["a", "b", "c"], "b", position=0) + assert removed is True + assert new_ids == ["a", "c"] + + +def test_remove_absent_id_reports_not_removed(): + new_ids, removed = remove_one_occurrence(["a", "b"], "zzz") + assert removed is False + assert new_ids == ["a", "b"] + + +def test_remove_single_occurrence(): + new_ids, removed = remove_one_occurrence(["a", "b", "c"], "b") + assert (new_ids, removed) == (["a", "c"], True) + + +def test_remove_stringifies(): + new_ids, removed = remove_one_occurrence([1, 2, 2, 3], 2) + assert removed and new_ids == ["1", "2", "3"] diff --git a/tests/test_playlist_reconcile.py b/tests/test_playlist_reconcile.py new file mode 100644 index 00000000..7e797ccc --- /dev/null +++ b/tests/test_playlist_reconcile.py @@ -0,0 +1,122 @@ +"""Extreme battery for the playlist sync-editor reconcile (#768). + +Covers: the reported YouTube failure (Bug A — "Artist - Title" source matching +its clean server copy instead of showing unmatched + orphan extra), the +source_track_id echo (Bug B), and parity with the original three-pass behavior +(override → exact → fuzzy → extra), plus duplicate-server-track handling. +""" + +from __future__ import annotations + +from core.sync.playlist_reconcile import norm_title, reconcile_playlist + + +def _src(name, artist, sid="", **kw): + return {"name": name, "artist": artist, "source_track_id": sid, **kw} + + +def _svr(title, artist, tid): + return {"title": title, "artist": artist, "id": tid, "ratingKey": tid} + + +def _status(combined): + return [(c["match_status"], + (c["source_track"] or {}).get("name"), + (c["server_track"] or {}).get("title")) for c in combined] + + +# ── Bug A: the reported YouTube case ────────────────────────────────────── + +def test_youtube_artist_title_source_matches_clean_server_track(): + source = [_src("Arctic Monkeys - Do I Wanna Know?", "Official Arctic Monkeys", "sp1")] + server = [_svr("Do I Wanna Know?", "Arctic Monkeys", "nv72")] + combined = reconcile_playlist(source, server) + assert len(combined) == 1 + assert combined[0]["match_status"] == "matched" + assert combined[0]["server_track"]["id"] == "nv72" + # ...and the server track is NOT left as an orphan extra. + assert not any(c["match_status"] == "extra" for c in combined) + + +def test_youtube_match_does_not_leave_unmatched_or_extra(): + # Before the fix this produced one 'missing' + one 'extra'. + source = [_src("The Killers - Mr. Brightside", "The KillersVEVO", "sp2")] + server = [_svr("Mr. Brightside", "The Killers", "nv5")] + statuses = [c["match_status"] for c in reconcile_playlist(source, server)] + assert statuses == ["matched"] + + +# ── Bug B: source_track_id is echoed back ───────────────────────────────── + +def test_source_track_id_present_on_matched_entry(): + source = [_src("Do I Wanna Know?", "Arctic Monkeys", "spotify:track:abc")] + server = [_svr("Do I Wanna Know?", "Arctic Monkeys", "nv1")] + combined = reconcile_playlist(source, server) + assert combined[0]["source_track"]["source_track_id"] == "spotify:track:abc" + + +def test_source_track_id_present_on_missing_entry(): + # A genuinely-missing source must still carry its id so it can be + # manually matched and persisted (the #768 manual-match loop). + source = [_src("Some Obscure B-Side", "Some Artist", "spotify:track:xyz")] + server = [_svr("Completely Different", "Other Artist", "nv9")] + combined = reconcile_playlist(source, server) + missing = [c for c in combined if c["match_status"] == "missing"] + assert missing and missing[0]["source_track"]["source_track_id"] == "spotify:track:xyz" + + +# ── parity: override / exact / fuzzy / extra ────────────────────────────── + +def test_override_pair_wins_first(): + source = [_src("Anything", "Whoever", "s1")] + server = [_svr("Totally Different Title", "Nobody", "nvX")] + combined = reconcile_playlist(source, server, override_pairs={0: 0}) + assert combined[0]["match_status"] == "matched" + assert combined[0]["confidence"] == 1.0 + assert combined[0].get("override") is True + + +def test_exact_normalized_match_strips_feat(): + source = [_src("Stay (feat. Justin Bieber)", "The Kid LAROI", "s1")] + server = [_svr("Stay", "The Kid LAROI", "nv1")] + assert reconcile_playlist(source, server)[0]["match_status"] == "matched" + + +def test_fuzzy_match_above_threshold(): + source = [_src("Mr Brightside", "The Killers", "s1")] + server = [_svr("Mr. Brightside", "The Killers", "nv1")] + c = reconcile_playlist(source, server)[0] + assert c["match_status"] == "matched" + assert c["confidence"] >= 0.75 + + +def test_truly_absent_track_is_missing_and_unrelated_server_is_extra(): + source = [_src("Nonexistent Song", "Ghost Artist", "s1")] + server = [_svr("Yellow", "Coldplay", "nv1")] + statuses = sorted(c["match_status"] for c in reconcile_playlist(source, server)) + assert statuses == ["extra", "missing"] + + +def test_each_server_track_claimed_once_no_double_match(): + # Two identical source rows must not both claim the single server track. + source = [_src("Yellow", "Coldplay", "s1"), _src("Yellow", "Coldplay", "s2")] + server = [_svr("Yellow", "Coldplay", "nv1")] + combined = reconcile_playlist(source, server) + matched = [c for c in combined if c["match_status"] == "matched"] + missing = [c for c in combined if c["match_status"] == "missing"] + assert len(matched) == 1 and len(missing) == 1 + + +def test_duplicate_server_tracks_one_matched_one_extra(): + # The #768 duplicate scenario: two copies of the same track on the server. + source = [_src("Do I Wanna Know?", "Arctic Monkeys", "s1")] + server = [_svr("Do I Wanna Know?", "Arctic Monkeys", "nv72"), + _svr("Do I Wanna Know?", "Arctic Monkeys", "nv73")] + combined = reconcile_playlist(source, server) + assert sorted(c["match_status"] for c in combined) == ["extra", "matched"] + + +def test_norm_title_helper_parity(): + assert norm_title("Stay (feat. X)") == "stay" + assert norm_title("Song (2019 Remaster)") == "song" + assert norm_title("Album (Deluxe Edition)") == "album" diff --git a/tests/test_source_title.py b/tests/test_source_title.py new file mode 100644 index 00000000..a53c8402 --- /dev/null +++ b/tests/test_source_title.py @@ -0,0 +1,113 @@ +"""Extreme test battery for source-track normalization (#768). + +YouTube/streaming sources carry "Artist - Song" titles and "Official Artist"/ +"Artist - Topic"/"ArtistVEVO" artist names; the library has clean metadata, so +matching fails and tracks are reported missing. These helpers strip the +decoration. The batteries below pin both the positives (must clean) and the +negatives (must NOT mangle real titles/artists). +""" + +from __future__ import annotations + +import pytest + +from core.text.source_title import ( + canonical_source_track, + clean_source_artist, + strip_artist_prefix, +) + + +# ── clean_source_artist ─────────────────────────────────────────────────── + +@pytest.mark.parametrize("raw,expected", [ + ("Official Arctic Monkeys", "Arctic Monkeys"), + ("The Official Weeknd", "Weeknd"), + ("Arctic Monkeys - Topic", "Arctic Monkeys"), + ("Coldplay - Topic", "Coldplay"), + ("ColdplayVEVO", "Coldplay"), + ("Coldplay VEVO", "Coldplay"), + ("EminemVEVO", "Eminem"), + (" Official Radiohead ", "Radiohead"), +]) +def test_clean_source_artist_strips_decoration(raw, expected): + assert clean_source_artist(raw) == expected + + +@pytest.mark.parametrize("raw", [ + "Arctic Monkeys", # already clean + "Coldplay", + "Twenty One Pilots", + "Death", + "U2", + "AJR", # would be emptied by a naive vevo/official strip + "", +]) +def test_clean_source_artist_leaves_clean_names(raw): + assert clean_source_artist(raw) == raw + + +def test_clean_source_artist_never_empties(): + # Pathological: artist that is ONLY decoration must not become "". + assert clean_source_artist("VEVO") == "VEVO" + assert clean_source_artist("Official ") == "Official" + + +# ── strip_artist_prefix ─────────────────────────────────────────────────── + +@pytest.mark.parametrize("title,artist,expected", [ + ("Arctic Monkeys - Do I Wanna Know?", "Arctic Monkeys", "Do I Wanna Know?"), + ("Death - Pull the Plug", "Death", "Pull the Plug"), + ("Coldplay – Yellow", "Coldplay", "Yellow"), # en dash + ("Coldplay — Yellow", "Coldplay", "Yellow"), # em dash + ("Eminem: Lose Yourself", "Eminem", "Lose Yourself"), # colon + ("Daft Punk | Get Lucky", "Daft Punk", "Get Lucky"), # pipe + ("ARCTIC MONKEYS - 505", "arctic monkeys", "505"), # case-fold +]) +def test_strip_artist_prefix_strips_when_prefix_is_artist(title, artist, expected): + assert strip_artist_prefix(title, artist) == expected + + +@pytest.mark.parametrize("title,artist", [ + ("Marvin Gaye", "Charlie Puth"), # title is not "artist - ..." + ("Do I Wanna Know?", "Arctic Monkeys"), # already clean + ("Self-Titled", "Whoever"), # hyphen w/o spaces — not a sep + ("Jay-Z Anthem", "Somebody"), # hyphen inside a word + ("Song - Live", "Coldplay"), # prefix "Song" != artist + ("Stay With Me", "Sam Smith"), + ("", "Arctic Monkeys"), + ("Arctic Monkeys -", "Arctic Monkeys"), # nothing after sep -> unchanged +]) +def test_strip_artist_prefix_leaves_others_untouched(title, artist): + assert strip_artist_prefix(title, artist) == title + + +def test_strip_only_first_separator(): + # "Artist - Song - Remix" -> strip only the leading artist segment. + assert strip_artist_prefix("Gorillaz - Feel Good Inc - Remix", "Gorillaz") == "Feel Good Inc - Remix" + + +# ── canonical_source_track (combined) ───────────────────────────────────── + +def test_canonical_handles_youtube_channel_and_prefix(): + # The reported case: channel-name artist + "Artist - Title" title. + title, artist = canonical_source_track( + "Arctic Monkeys - Do I Wanna Know?", "Official Arctic Monkeys", + ) + assert title == "Do I Wanna Know?" + assert artist == "Arctic Monkeys" + + +def test_canonical_strips_prefix_using_raw_artist_when_clean_differs(): + # Title prefixed with the channel-style raw artist itself. + title, artist = canonical_source_track( + "Official Arctic Monkeys - 505", "Official Arctic Monkeys", + ) + # cleaned artist is "Arctic Monkeys"; raw prefix "Official Arctic Monkeys" + # also stripped via the raw-artist fallback. + assert title == "505" + assert artist == "Arctic Monkeys" + + +def test_canonical_noop_on_clean_input(): + assert canonical_source_track("Yellow", "Coldplay") == ("Yellow", "Coldplay") diff --git a/web_server.py b/web_server.py index 9572dfe8..dbb363e1 100644 --- a/web_server.py +++ b/web_server.py @@ -18185,23 +18185,11 @@ def get_server_playlist_tracks(playlist_id): pass break - # Build combined view with two-pass matching (exact then fuzzy) - import re as _re - from difflib import SequenceMatcher - - def _norm_title(t): - """Strip feat./ft., remaster, and edition qualifiers for comparison only.""" - # feat./ft. — e.g. (feat. Artist), [ft. Artist] - t = _re.sub(r'\s*[\(\[](?:feat|ft)\.?[^\)\]]*[\)\]]', '', t, flags=_re.IGNORECASE) - # Remaster/Remastered — e.g. (2019 Remaster), (Remastered), (2019 Remastered Version) - t = _re.sub(r'\s*[\(\[](?:\d{4}\s+)?remaster(?:ed)?(?:\s+version)?\s*[\)\]]', '', t, flags=_re.IGNORECASE) - # Edition qualifiers — e.g. (Deluxe Edition), (Special Edition), [Anniversary Edition] - t = _re.sub(r'\s*[\(\[](?:deluxe|special|anniversary|legacy|expanded|limited)(?:\s+edition)?\s*[\)\]]', '', t, flags=_re.IGNORECASE) - return t.lower().strip() - - combined = [] - used_server_indices = set() - unmatched_source = [] # (index_in_combined, src_dict) for fuzzy second pass + # Reconcile source vs server playlist. Three-pass matcher lifted to + # core.sync.playlist_reconcile (pure + tested) — fixes #768 (YouTube + # "Artist - Title" sources now match, and source_track_id is echoed + # back so manual "Find & add" overrides persist). + from core.sync.playlist_reconcile import reconcile_playlist # Pass 0: User-confirmed match overrides from sync_match_cache. # When a user previously picked a local file via "Find & Add", @@ -18218,92 +18206,7 @@ def get_server_playlist_tracks(playlist_id): lambda src_id: ((_db_for_overrides.read_sync_match_cache(src_id, active_server) or {}).get('server_track_id')), ) - # Pass 1: Exact title match (normalized — strips feat./ft. qualifiers) - for i, src in enumerate(source_tracks): - src_name = src.get('name', '') - src_artist = src.get('artist', '') - if not src_artist and src.get('artists'): - a = src['artists'][0] if src['artists'] else '' - src_artist = a.get('name', a) if isinstance(a, dict) else str(a) - - src_entry = { - 'name': src_name, 'artist': src_artist, - 'album': src.get('album', ''), 'image_url': src.get('image_url', ''), - 'duration_ms': src.get('duration_ms', 0), 'position': src.get('position', i), - } - - # Override hit — paired by user, skip exact/fuzzy matching. - if i in _override_pairs: - j_override = _override_pairs[i] - used_server_indices.add(j_override) - combined.append({ - 'source_track': src_entry, - 'server_track': server_tracks[j_override], - 'match_status': 'matched', - 'confidence': 1.0, - 'override': True, - }) - continue - - src_norm = _norm_title(src_name) - best_idx = -1 - for j, svr in enumerate(server_tracks): - if j in used_server_indices: - continue - if _norm_title(svr['title']) == src_norm: - best_idx = j - break - - if best_idx >= 0: - used_server_indices.add(best_idx) - combined.append({ - 'source_track': src_entry, - 'server_track': server_tracks[best_idx], - 'match_status': 'matched', - 'confidence': 1.0, - }) - else: - idx = len(combined) - combined.append({ - 'source_track': src_entry, - 'server_track': None, - 'match_status': 'missing', - 'confidence': 0.0, - }) - unmatched_source.append((idx, src_entry)) - - # Pass 2: Fuzzy match on remaining unmatched source tracks (normalized keys) - for combo_idx, src_entry in unmatched_source: - src_key = f"{src_entry['artist']} {_norm_title(src_entry['name'])}".strip() - best_score = 0.0 - best_j = -1 - for j, svr in enumerate(server_tracks): - if j in used_server_indices: - continue - svr_key = f"{svr['artist']} {_norm_title(svr['title'])}".strip().lower() - score = SequenceMatcher(None, src_key.lower(), svr_key).ratio() - if score > best_score and score >= 0.75: - best_score = score - best_j = j - - if best_j >= 0: - used_server_indices.add(best_j) - combined[combo_idx] = { - 'source_track': src_entry, - 'server_track': server_tracks[best_j], - 'match_status': 'matched', - 'confidence': round(best_score, 3), - } - - # Add server tracks that aren't in the source (extra tracks on server) - for j, svr in enumerate(server_tracks): - if j not in used_server_indices: - combined.append({ - 'source_track': None, - 'server_track': svr, - 'match_status': 'extra', - 'confidence': 0.0, - }) + combined = reconcile_playlist(source_tracks, server_tracks, _override_pairs) return jsonify({ "success": True, @@ -18490,6 +18393,18 @@ def server_playlist_add_track(playlist_id): if not new_item: return jsonify({"success": False, "error": "Track not found on server"}), 404 + # Link, don't duplicate: matching an unmatched source to a track + # already in the playlist should only record the override, never + # append a second copy (#768). + if source_track_id: + try: + _existing = {str(it.ratingKey) for it in raw_playlist.items()} + except Exception: + _existing = set() + if str(track_id) in _existing: + _persist_find_and_add_match(source_track_id, active_server, track_id, server_track_title or new_item.title, source_title, source_artist) + return jsonify({"success": True, "message": "Track linked"}) + logger.info(f"[ServerPlaylist] Adding track: '{new_item.title}' (ratingKey={new_item.ratingKey}) to playlist '{playlist_name}'") raw_playlist.addItems([new_item]) @@ -18515,24 +18430,30 @@ def server_playlist_add_track(playlist_id): return jsonify({"success": True, "message": "Track added", "new_playlist_id": new_id}) elif active_server == 'jellyfin' and media_server_engine.client('jellyfin'): + from core.sync.playlist_edit import plan_playlist_add current_tracks = media_server_engine.client('jellyfin').get_playlist_tracks(playlist_id) or [] track_ids = [str(t.ratingKey) for t in current_tracks] - pos = max(0, min(int(position), len(track_ids))) if position is not None else len(track_ids) - track_ids.insert(pos, track_id) - new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in track_ids] - media_server_engine.client('jellyfin').update_playlist(playlist_name, new_track_objs) + # Matching an unmatched source to a track already in the playlist + # is a LINK, not a second copy — don't duplicate it (#768). + plan = plan_playlist_add(track_ids, track_id, is_link=bool(source_track_id), position=position) + if plan['should_insert']: + new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in plan['new_ids']] + media_server_engine.client('jellyfin').update_playlist(playlist_name, new_track_objs) _persist_find_and_add_match(source_track_id, active_server, track_id, server_track_title, source_title, source_artist) - return jsonify({"success": True, "message": "Track added"}) + return jsonify({"success": True, "message": "Track linked" if not plan['should_insert'] else "Track added"}) elif active_server == 'navidrome' and media_server_engine.client('navidrome'): + from core.sync.playlist_edit import plan_playlist_add current_tracks = media_server_engine.client('navidrome').get_playlist_tracks(playlist_id) or [] track_ids = [str(t.ratingKey) for t in current_tracks] - pos = max(0, min(int(position), len(track_ids))) if position is not None else len(track_ids) - track_ids.insert(pos, track_id) - new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in track_ids] - media_server_engine.client('navidrome').create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) + # Matching an unmatched source to a track already in the playlist + # is a LINK, not a second copy — don't duplicate it (#768). + plan = plan_playlist_add(track_ids, track_id, is_link=bool(source_track_id), position=position) + if plan['should_insert']: + new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in plan['new_ids']] + media_server_engine.client('navidrome').create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) _persist_find_and_add_match(source_track_id, active_server, track_id, server_track_title, source_title, source_artist) - return jsonify({"success": True, "message": "Track added"}) + return jsonify({"success": True, "message": "Track linked" if not plan['should_insert'] else "Track added"}) return jsonify({"success": False, "error": f"Unsupported server: {active_server}"}), 400 except Exception as e: @@ -18573,10 +18494,17 @@ def server_playlist_remove_track(playlist_id): logger.warning(f"[ServerPlaylist] remove-track: playlist not found by id={playlist_id} or name='{playlist_name}'") return jsonify({"success": False, "error": "Playlist not found"}), 404 - # Rebuild without the target track + # Rebuild without ONE copy of the target track — deleting one + # duplicate must not wipe every copy (#768). current_items = list(raw_playlist.items()) - new_items = [item for item in current_items if str(item.ratingKey) != str(remove_track_id)] - if len(new_items) == len(current_items): + new_items = list(current_items) + _removed = False + for _i, _it in enumerate(new_items): + if str(_it.ratingKey) == str(remove_track_id): + del new_items[_i] + _removed = True + break + if not _removed: return jsonify({"success": False, "error": "Track not found in playlist"}), 404 raw_playlist.delete() if new_items: @@ -18586,18 +18514,25 @@ def server_playlist_remove_track(playlist_id): return jsonify({"success": True, "message": "Track removed (playlist now empty)"}) elif active_server == 'jellyfin' and media_server_engine.client('jellyfin'): + from core.sync.playlist_edit import remove_one_occurrence current_tracks = media_server_engine.client('jellyfin').get_playlist_tracks(playlist_id) or [] - new_ids = [str(t.ratingKey) for t in current_tracks if str(t.ratingKey) != str(remove_track_id)] - if len(new_ids) == len(current_tracks): + track_ids = [str(t.ratingKey) for t in current_tracks] + # Remove ONE occurrence, not every copy — duplicates are the same + # track, so deleting one must not wipe them all (#768). + new_ids, removed = remove_one_occurrence(track_ids, remove_track_id) + if not removed: return jsonify({"success": False, "error": "Track not found in playlist"}), 404 new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in new_ids] media_server_engine.client('jellyfin').update_playlist(playlist_name, new_track_objs) return jsonify({"success": True, "message": "Track removed"}) elif active_server == 'navidrome' and media_server_engine.client('navidrome'): + from core.sync.playlist_edit import remove_one_occurrence current_tracks = media_server_engine.client('navidrome').get_playlist_tracks(playlist_id) or [] - new_ids = [str(t.ratingKey) for t in current_tracks if str(t.ratingKey) != str(remove_track_id)] - if len(new_ids) == len(current_tracks): + track_ids = [str(t.ratingKey) for t in current_tracks] + # Remove ONE occurrence, not every copy (#768). + new_ids, removed = remove_one_occurrence(track_ids, remove_track_id) + if not removed: return jsonify({"success": False, "error": "Track not found in playlist"}), 404 new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in new_ids] media_server_engine.client('navidrome').create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) From 3b49ac8280b267fc554ce694570317ea5f7f0d93 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 10:32:06 -0700 Subject: [PATCH 006/108] Fix #767: Library Organizer dry run no longer creates folders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reorganize preview (dry run) was physically creating destination album folders, littering the library with empty dirs and making "changes" before the user ever hit Apply. Cause: preview_album_reorganize calls build_final_path_for_track purely to COMPUTE the destination path string — but that shared helper has 9 os.makedirs side effects (it's also the live download/import path builder, where creating the dir is correct). So computing the preview path created "Lenka (Expanded Edition)/" on disk. Fix: build_final_path_for_track gains create_dirs=True; all 9 makedirs now route through a gated helper. The reorganize PREVIEW passes create_dirs=False, so a dry run computes the exact destination path with zero filesystem side effects. Everything else keeps the default True: - the download/import post-process flow (still writes files into the dir), - retag, - the reorganize APPLY path — verified it goes through post_process_fn (the real pipeline → build_final_path_for_track with create_dirs=True), so live moves still create their destination dirs. The gate only silences the dry run. Tests: tests/imports/test_import_paths.py — create_dirs=False computes the correct path (matching the reported "01 - The Show.flac") but writes NOTHING to disk (not even the Transfer root); create_dirs=True still creates folders; both yield an identical path. Updated two reorganize-orchestrator test doubles to accept the new kwarg. 148 reorganize/paths/retag/pipeline tests pass. Does NOT fix the second half of #767 (Expanded Edition picked over the standard album). That is NOT a reorganizer bug: the library album row was linked to the deluxe release at enrichment time (its stored spotify_album_id/itunes_album_id/ deezer_id points at "Lenka (Expanded Edition)"), and the reorganizer faithfully reorganizes to whatever the album is linked to. The real fix is in album enrichment's edition preference — tracked separately. --- core/imports/paths.py | 34 +++++--- core/library_reorganize.py | 7 +- tests/imports/test_import_paths.py | 78 +++++++++++++++++++ tests/test_library_reorganize_orchestrator.py | 7 +- 4 files changed, 111 insertions(+), 15 deletions(-) diff --git a/core/imports/paths.py b/core/imports/paths.py index af9619d7..5744744e 100644 --- a/core/imports/paths.py +++ b/core/imports/paths.py @@ -418,8 +418,20 @@ def _coerce_int(value: Any, default: int = 1) -> int: return coerced if coerced > 0 else default -def build_final_path_for_track(context, artist_context, album_info, file_ext): - """Shared path builder used by both post-processing and verification.""" +def build_final_path_for_track(context, artist_context, album_info, file_ext, create_dirs: bool = True): + """Shared path builder used by both post-processing and verification. + + ``create_dirs`` gates the directory-creation side effects. The download + import flow leaves it True (it's about to write the file there). The + library-reorganize PREVIEW passes False so a dry run can compute the exact + destination path WITHOUT physically creating the folder — fixes #767 (dry + run was leaving empty destination folders behind).""" + _real_makedirs = os.makedirs + + def _ensure_dir(path, **_kw): + if create_dirs: + _real_makedirs(path, exist_ok=True) + transfer_dir = docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer")) context = normalize_import_context(context) track_info = get_import_track_info(context) @@ -440,7 +452,7 @@ def build_final_path_for_track(context, artist_context, album_info, file_ext): original_dir = os.path.dirname(original_path) original_stem = os.path.splitext(os.path.basename(original_path))[0] final_path = os.path.join(original_dir, original_stem + file_ext) - os.makedirs(original_dir, exist_ok=True) + _ensure_dir(original_dir, exist_ok=True) logger.info("[Enhance] Using original file location: %s", final_path) return final_path, True @@ -477,12 +489,12 @@ def build_final_path_for_track(context, artist_context, album_info, file_ext): folder_path, filename_base = get_file_path_from_template(template_context, "playlist_path") if folder_path and filename_base: final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) - os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + _ensure_dir(os.path.join(transfer_dir, folder_path), exist_ok=True) return final_path, True playlist_name_sanitized = sanitize_filename(playlist_name) playlist_dir = os.path.join(transfer_dir, playlist_name_sanitized) - os.makedirs(playlist_dir, exist_ok=True) + _ensure_dir(playlist_dir, exist_ok=True) artist_name_sanitized = sanitize_filename(template_context["artist"]) track_name_sanitized = sanitize_filename(track_name) new_filename = f"{artist_name_sanitized} - {track_name_sanitized}{file_ext}" @@ -579,10 +591,10 @@ def build_final_path_for_track(context, artist_context, album_info, file_ext): if total_discs > 1 and not user_controls_disc: disc_folder = f"{disc_label} {disc_number}" final_path = os.path.join(transfer_dir, folder_path, disc_folder, filename_base + file_ext) - os.makedirs(os.path.join(transfer_dir, folder_path, disc_folder), exist_ok=True) + _ensure_dir(os.path.join(transfer_dir, folder_path, disc_folder), exist_ok=True) else: final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) - os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + _ensure_dir(os.path.join(transfer_dir, folder_path), exist_ok=True) return final_path, True artist_name_sanitized = sanitize_filename(template_context["albumartist"]) @@ -592,7 +604,7 @@ def build_final_path_for_track(context, artist_context, album_info, file_ext): album_dir = os.path.join(artist_dir, album_folder_name) if total_discs > 1: album_dir = os.path.join(album_dir, f"{disc_label} {disc_number}") - os.makedirs(album_dir, exist_ok=True) + _ensure_dir(album_dir, exist_ok=True) final_track_name_sanitized = sanitize_filename(clean_track_name) new_filename = f"{track_number:02d} - {final_track_name_sanitized}{file_ext}" return os.path.join(album_dir, new_filename), True @@ -629,10 +641,10 @@ def build_final_path_for_track(context, artist_context, album_info, file_ext): if filename_base: if folder_path: final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) - os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + _ensure_dir(os.path.join(transfer_dir, folder_path), exist_ok=True) else: final_path = os.path.join(transfer_dir, filename_base + file_ext) - os.makedirs(transfer_dir, exist_ok=True) + _ensure_dir(transfer_dir, exist_ok=True) return final_path, True artist_name_sanitized = sanitize_filename(template_context["artist"]) @@ -640,6 +652,6 @@ def build_final_path_for_track(context, artist_context, album_info, file_ext): artist_dir = os.path.join(transfer_dir, artist_name_sanitized) single_folder_name = f"{artist_name_sanitized} - {final_track_name_sanitized}" single_dir = os.path.join(artist_dir, single_folder_name) - os.makedirs(single_dir, exist_ok=True) + _ensure_dir(single_dir, exist_ok=True) new_filename = f"{final_track_name_sanitized}{file_ext}" return os.path.join(single_dir, new_filename), True diff --git a/core/library_reorganize.py b/core/library_reorganize.py index af795789..25adcda4 100644 --- a/core/library_reorganize.py +++ b/core/library_reorganize.py @@ -989,7 +989,12 @@ def preview_album_reorganize( album_info = _build_album_info(context) try: spotify_artist = context['spotify_artist'] - new_full, _ok = build_final_path_fn(context, spotify_artist, album_info, file_ext) + # Dry run: compute the destination path WITHOUT creating the folder. + # Previously this physically created the album dir during preview, + # leaving empty folders all over the library (#767). + new_full, _ok = build_final_path_fn( + context, spotify_artist, album_info, file_ext, create_dirs=False + ) item['new_path'] = ( os.path.relpath(new_full, transfer_dir) if transfer_dir and new_full and new_full.startswith(transfer_dir) diff --git a/tests/imports/test_import_paths.py b/tests/imports/test_import_paths.py index ec94881d..ee4fd1b6 100644 --- a/tests/imports/test_import_paths.py +++ b/tests/imports/test_import_paths.py @@ -17,6 +17,84 @@ def test_sanitize_filename_replaces_illegal_characters(): assert import_paths.sanitize_filename("AUX.txt").startswith("_") +# ── #767: dry-run path build must not create folders ────────────────────── + +def _album_path_config(tmp_path): + return _Config({ + "soulseek.transfer_path": str(tmp_path / "Transfer"), + "file_organization.enabled": True, + "file_organization.templates": { + "album_path": "$albumartist/$albumartist - $album/$track - $title", + "single_path": "$artist/$artist - $title", + }, + "file_organization.collab_artist_mode": "first", + "file_organization.disc_label": "Disc", + }) + + +def _album_context(): + return { + "artist": {"name": "Lenka"}, + "album": {"name": "Lenka", "id": "album-1", "release_date": "2008-01-01", + "total_tracks": 12, "album_type": "album", "artists": [{"name": "Lenka"}]}, + "track_info": {"name": "The Show", "id": "t1", "track_number": 1, + "disc_number": 1, "artists": [{"name": "Lenka"}]}, + "original_search_result": {"title": "The Show", "clean_title": "The Show", + "clean_album": "Lenka", "clean_artist": "Lenka", + "artists": [{"name": "Lenka"}]}, + "source": "deezer", "is_album_download": False, + } + + +def test_create_dirs_false_does_not_create_folders(monkeypatch, tmp_path): + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _album_path_config(tmp_path)) + monkeypatch.setattr(import_paths, "_get_album_tracks_for_source", lambda *a: None) + + final_path, created = import_paths.build_final_path_for_track( + _album_context(), {"name": "Lenka"}, + {"is_album": True, "album_name": "Lenka", "track_number": 1, "disc_number": 1}, + ".flac", create_dirs=False, + ) + + # Path is still computed correctly... + assert created is True + assert final_path == str(tmp_path / "Transfer" / "Lenka" / "Lenka - Lenka" / "01 - The Show.flac") + # ...but NOTHING was written to disk — not even the Transfer root. + assert not (tmp_path / "Transfer").exists() + + +def test_create_dirs_true_still_creates_folders(monkeypatch, tmp_path): + # The download/import flow must keep working (default behavior unchanged). + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _album_path_config(tmp_path)) + monkeypatch.setattr(import_paths, "_get_album_tracks_for_source", lambda *a: None) + + final_path, created = import_paths.build_final_path_for_track( + _album_context(), {"name": "Lenka"}, + {"is_album": True, "album_name": "Lenka", "track_number": 1, "disc_number": 1}, + ".flac", # create_dirs defaults True + ) + + assert created is True + assert (tmp_path / "Transfer" / "Lenka" / "Lenka - Lenka").is_dir() + + +def test_create_dirs_false_and_true_yield_identical_path(monkeypatch, tmp_path): + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _album_path_config(tmp_path)) + monkeypatch.setattr(import_paths, "_get_album_tracks_for_source", lambda *a: None) + + dry, _ = import_paths.build_final_path_for_track( + _album_context(), {"name": "Lenka"}, + {"is_album": True, "album_name": "Lenka", "track_number": 1, "disc_number": 1}, + ".flac", create_dirs=False, + ) + live, _ = import_paths.build_final_path_for_track( + _album_context(), {"name": "Lenka"}, + {"is_album": True, "album_name": "Lenka", "track_number": 1, "disc_number": 1}, + ".flac", create_dirs=True, + ) + assert dry == live + + def test_build_simple_download_destination_uses_album_folder(monkeypatch, tmp_path): config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")}) monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) diff --git a/tests/test_library_reorganize_orchestrator.py b/tests/test_library_reorganize_orchestrator.py index e4045b9f..0b068a7a 100644 --- a/tests/test_library_reorganize_orchestrator.py +++ b/tests/test_library_reorganize_orchestrator.py @@ -1163,9 +1163,10 @@ def test_keeps_album_sidecars_when_a_track_failed_to_move(monkeypatch, tmpdirs): # --- preview function (shared planning with the orchestrator) ----------- -def _fake_path_builder(context, spotify_artist, _album_info, file_ext): +def _fake_path_builder(context, spotify_artist, _album_info, file_ext, **_kw): """Stand-in for `_build_final_path_for_track`. Inserts Disc N/ when - total_discs > 1 — same convention the real builder uses.""" + total_discs > 1 — same convention the real builder uses. Accepts + **_kw so the preview's create_dirs=False kwarg (#767) passes through.""" album = context['spotify_album']['name'] artist = spotify_artist['name'] track_info = context['track_info'] @@ -1180,7 +1181,7 @@ def _fake_path_builder(context, spotify_artist, _album_info, file_ext): return '/'.join(parts), True -def _path_builder_album_vs_single(context, spotify_artist, album_info, file_ext): +def _path_builder_album_vs_single(context, spotify_artist, album_info, file_ext, **_kw): """Stand-in that emulates the real `_build_final_path_for_track` branch on `album_info.get('is_album')`. ALBUM mode produces an album folder with disc subfolder + numbered file; SINGLE mode From 89b438974f4d67d4f66d78518ce335c7c8b57c59 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 11:01:28 -0700 Subject: [PATCH 007/108] Fix #766: Navidrome album covers blank in the sync editor (+ other modals) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sync editor renders server covers as , but no Flask route ever served that path — so every Navidrome cover 404'd, on every album, art or not. The source (left) side then went blank too: a source row with no native art (e.g. YouTube, which provides none at mirror time) falls back to borrowing the matched server track's cover — i.e. that same dead route. So both sides collapsed to nothing. Fix: - New NavidromeClient.build_cover_art_url(cover_id) — builds the absolute, Subsonic-authenticated getCoverArt URL (base_url + token/salt), keeping credentials server-side. Uses a FIXED cover-art salt so the URL is deterministic for a given (server, password, cover_id): a rotating salt (as in _generate_auth_params) would make every request a unique URL → image-cache miss every time + a dead, never-reused cache row per fetch. Token auth doesn't require a unique salt, and the password is never exposed (only its salted md5). - New route /api/navidrome/cover/ — resolves that URL and streams the image through the shared image cache (same pattern as /api/image-proxy), with a private max-age so the browser caches by the stable route URL. Effect: server side works for any album that has art in Navidrome; matched source rows with no native art now borrow the (now-working) server cover. Unmatched YouTube rows stay blank — no image exists anywhere to show. Tests: tests/test_navidrome_cover_url.py (8) — URL structure + salted-token auth (never the raw password), determinism (same id -> same URL so the cache hits; different id/password -> different URL), optional size, and the not-connected / no-id / no-credentials guards. Caveats: not executed against a live Navidrome (no server in CI) — the URL builder is unit-tested; the route's cache→HTTP→bytes round-trip is read-verified only. Scope is the sync editor's Navidrome route; Plex/Jellyfin server-cover branches and any other modals using a different mechanism are untouched. --- core/navidrome_client.py | 37 ++++++++++++++ tests/test_navidrome_cover_url.py | 83 +++++++++++++++++++++++++++++++ web_server.py | 27 ++++++++++ 3 files changed, 147 insertions(+) create mode 100644 tests/test_navidrome_cover_url.py diff --git a/core/navidrome_client.py b/core/navidrome_client.py index 7ec9d286..765153bc 100644 --- a/core/navidrome_client.py +++ b/core/navidrome_client.py @@ -3,6 +3,7 @@ import hashlib import secrets from typing import List, Optional, Dict, Any from datetime import datetime +from urllib.parse import urlencode import json from utils.logging_config import get_logger from config.settings import config_manager @@ -316,6 +317,42 @@ class NavidromeClient(MediaServerClient): 'f': 'json' # Response format } + # Fixed salt for cover-art URLs ONLY. Subsonic token auth (t=md5(password + # +salt), s=salt) does not require a unique salt per request, and a stable + # one makes the cover URL deterministic — so the image cache and the + # browser cache actually HIT. The rotating salt from _generate_auth_params + # would make every request a unique URL → cache miss every time + a dead, + # never-reused cache row per fetch (#766 review). The password is never + # exposed either way (only its salted md5). + _COVER_ART_SALT = 'soulsync-cover' + + def build_cover_art_url(self, cover_id, size=None) -> Optional[str]: + """Absolute, Subsonic-authenticated getCoverArt URL for ``cover_id``. + + Deterministic for a given (server, password, cover_id) so it caches. + The web layer proxies this to the browser (sync editor + modals). + Returns ``None`` when not connected or no id was supplied. #766: the + ``/api/navidrome/cover/`` route had no working URL behind it, so + every Navidrome cover came back blank.""" + if not self.base_url or not cover_id: + return None + if not self.username or not self.password: + return None + salt = self._COVER_ART_SALT + token = hashlib.md5((self.password + salt).encode()).hexdigest() + params = { + 'u': self.username, + 't': token, + 's': salt, + 'v': '1.16.1', + 'c': 'SoulSync', + 'f': 'json', # harmless for getCoverArt — it returns image binary + 'id': str(cover_id), + } + if size: + params['size'] = str(size) + return f"{self.base_url}/rest/getCoverArt?{urlencode(params)}" + # Subsonic endpoints that modify data — use POST to avoid URL length limits _WRITE_ENDPOINTS = frozenset({ 'createPlaylist', 'updatePlaylist', 'deletePlaylist', diff --git a/tests/test_navidrome_cover_url.py b/tests/test_navidrome_cover_url.py new file mode 100644 index 00000000..67894bbb --- /dev/null +++ b/tests/test_navidrome_cover_url.py @@ -0,0 +1,83 @@ +"""Tests for Navidrome cover-art URL building (#766). + +The sync editor + modals referenced /api/navidrome/cover/ but no route +served it, and the URL behind it had to be a fully-authenticated Subsonic +getCoverArt URL. build_cover_art_url is that builder — these pin its shape and +the not-connected guards (the token/salt are random per call, so we assert +structure + required params rather than an exact string). +""" + +from __future__ import annotations + +from urllib.parse import parse_qs, urlsplit + +from core.navidrome_client import NavidromeClient + + +def _connected_client(): + c = NavidromeClient() + c.base_url = "https://nav.example.com" + c.username = "boulder" + c.password = "hunter2" + return c + + +def test_builds_authenticated_cover_url(): + url = _connected_client().build_cover_art_url("al-123") + parts = urlsplit(url) + assert parts.scheme == "https" + assert parts.netloc == "nav.example.com" + assert parts.path == "/rest/getCoverArt" + q = parse_qs(parts.query) + assert q["id"] == ["al-123"] + assert q["u"] == ["boulder"] + # Subsonic token auth: salted md5, never the raw password. + assert q["t"] and q["t"][0] != "hunter2" + assert q["s"] # salt present + assert "hunter2" not in url + for required in ("t", "s", "v", "c"): + assert required in q + + +def test_cover_url_is_deterministic_so_the_cache_hits(): + # #766 review: the URL must be stable for a given (server, password, + # cover_id) — otherwise the image cache keys on a rotating salt and misses + # every request, re-fetching Navidrome each time + leaking dead rows. + c = _connected_client() + assert c.build_cover_art_url("al-123") == c.build_cover_art_url("al-123") + # ...and different covers still produce different URLs. + assert c.build_cover_art_url("al-123") != c.build_cover_art_url("al-999") + + +def test_cover_url_changes_with_password(): + # A password change must invalidate the cached URL (new token). + c1 = _connected_client() + c2 = _connected_client() + c2.password = "different" + assert c1.build_cover_art_url("al-1") != c2.build_cover_art_url("al-1") + + +def test_size_param_optional(): + assert "size" not in (_connected_client().build_cover_art_url("x") or "") + assert "size=300" in _connected_client().build_cover_art_url("x", size=300) + + +def test_cover_id_is_stringified(): + url = _connected_client().build_cover_art_url(12345) + assert "id=12345" in url + + +def test_returns_none_when_not_connected(): + c = NavidromeClient() # base_url is None + assert c.build_cover_art_url("al-1") is None + + +def test_returns_none_for_empty_cover_id(): + assert _connected_client().build_cover_art_url("") is None + assert _connected_client().build_cover_art_url(None) is None + + +def test_returns_none_without_credentials(): + c = NavidromeClient() + c.base_url = "https://nav.example.com" # but no username/password + assert c.build_cover_art_url("al-1") is None diff --git a/web_server.py b/web_server.py index dbb363e1..2849ac1c 100644 --- a/web_server.py +++ b/web_server.py @@ -4133,6 +4133,33 @@ def select_jellyfin_music_library(): logger.error(f"Error setting Jellyfin music library: {e}") return jsonify({"success": False, "error": str(e)}), 500 +@app.route('/api/navidrome/cover/', methods=['GET']) +def navidrome_cover(cover_id): + """Proxy a Navidrome (Subsonic) cover-art image to the browser. + + The sync editor and other modals reference /api/navidrome/cover/, + but no route served it — so every Navidrome cover came back blank (#766). + We build the authenticated getCoverArt URL server-side (keeping Subsonic + credentials off the client) and stream it through the shared image cache. + """ + try: + client = media_server_engine.client('navidrome') + if not client: + return '', 404 + url = client.build_cover_art_url(cover_id) + if not url: + return '', 404 + from core.image_cache import get_image_cache + cached = get_image_cache().get_url(url) + response = send_file(cached.path, mimetype=cached.mime_type, conditional=True) + max_age = int(config_manager.get("image_cache.ttl_seconds", 2592000)) + response.headers['Cache-Control'] = f'private, max-age={max_age}' + return response + except Exception as exc: + logger.debug("navidrome cover proxy failed for %s: %s", cover_id, exc) + return '', 502 + + @app.route('/api/navidrome/music-folders', methods=['GET']) def get_navidrome_music_folders(): """Get list of available music folders from Navidrome""" From cd9e4abc7c54c9fdc41b2529366a5b4d2ba269cb Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 11:08:35 -0700 Subject: [PATCH 008/108] #766 follow-on: source rows borrow their matched server track's cover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A source row with no art of its own (e.g. a YouTube source, which provides none at mirror time) now borrows the cover from its MATCHED server track, so both sides of the sync editor show an image. The endpoint already had a borrow fallback (_server_art_map), but it matched by an exact normalized "{artist}|{title}" key — so a YouTube-shaped row like "Arctic Monkeys - Do I Wanna Know?" never matched the library's "Do I Wanna Know?" and stayed blank even though the server had the cover. This borrow is keyed off the ACTUAL source<->server pairing the reconcile already computed, so it works for those rows once #768's canonical matching pairs them. Done in the pure reconcile_playlist (final pass), so no frontend change is needed — the editor already renders source_track.image_url. Guarded so it only fills an EMPTY source image (Spotify/CDN art is never overwritten) and only when the matched server track actually has a thumb. Composes with the rest: #766 made the server cover URL work, #768 made the YouTube row match, this makes the matched source row borrow that cover — so an artless YouTube row matched to a Navidrome track with art shows on both sides. Tests: tests/test_playlist_reconcile.py (+4) — artless source borrows the matched cover; source with its own art keeps it; unmatched source has nothing to borrow; borrow skipped when the server track has no thumb. 15 reconcile + 59 sync/navidrome tests pass. --- core/sync/playlist_reconcile.py | 11 +++++++++ tests/test_playlist_reconcile.py | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/core/sync/playlist_reconcile.py b/core/sync/playlist_reconcile.py index 8a37fafc..eb402b6d 100644 --- a/core/sync/playlist_reconcile.py +++ b/core/sync/playlist_reconcile.py @@ -180,6 +180,17 @@ def reconcile_playlist( 'confidence': 0.0, }) + # #766: a source row with no art of its own (e.g. a YouTube source, which + # provides none) borrows its MATCHED server track's cover so both sides of + # the editor show an image. Keyed off the actual pairing — works for + # "Artist - Title" rows that a fuzzy title lookup would miss. Source rows + # that already have their own art (Spotify CDN, etc.) keep it. + for entry in combined: + st = entry.get('source_track') + sv = entry.get('server_track') + if st and sv and not st.get('image_url') and sv.get('thumb'): + st['image_url'] = sv['thumb'] + return combined diff --git a/tests/test_playlist_reconcile.py b/tests/test_playlist_reconcile.py index 7e797ccc..0966f74c 100644 --- a/tests/test_playlist_reconcile.py +++ b/tests/test_playlist_reconcile.py @@ -116,6 +116,46 @@ def test_duplicate_server_tracks_one_matched_one_extra(): assert sorted(c["match_status"] for c in combined) == ["extra", "matched"] +# ── #766: source borrows the matched server cover ───────────────────────── + +def _svr_art(title, artist, tid, thumb): + return {"title": title, "artist": artist, "id": tid, "ratingKey": tid, "thumb": thumb} + + +def test_artless_source_borrows_matched_server_cover(): + # YouTube-style source row, no art of its own, matched to a server track + # that has a cover -> source side borrows it. + source = [_src("Arctic Monkeys - Do I Wanna Know?", "Official Arctic Monkeys", "s1")] + server = [_svr_art("Do I Wanna Know?", "Arctic Monkeys", "nv1", "/api/navidrome/cover/al42")] + combined = reconcile_playlist(source, server) + assert combined[0]["match_status"] == "matched" + assert combined[0]["source_track"]["image_url"] == "/api/navidrome/cover/al42" + + +def test_source_keeps_its_own_art_when_present(): + # A Spotify-style source row with its own CDN art must NOT be overwritten. + source = [_src("Do I Wanna Know?", "Arctic Monkeys", "s1", image_url="https://cdn/spotify.jpg")] + server = [_svr_art("Do I Wanna Know?", "Arctic Monkeys", "nv1", "/api/navidrome/cover/al42")] + combined = reconcile_playlist(source, server) + assert combined[0]["source_track"]["image_url"] == "https://cdn/spotify.jpg" + + +def test_unmatched_source_has_no_cover_to_borrow(): + source = [_src("Totally Absent Song", "Ghost", "s1")] + server = [_svr_art("Something Else", "Nobody", "nv1", "/api/navidrome/cover/al99")] + combined = reconcile_playlist(source, server) + missing = [c for c in combined if c["match_status"] == "missing"] + assert missing and not missing[0]["source_track"]["image_url"] + + +def test_borrow_skipped_when_server_track_has_no_thumb(): + source = [_src("Do I Wanna Know?", "Arctic Monkeys", "s1")] + server = [_svr("Do I Wanna Know?", "Arctic Monkeys", "nv1")] # no thumb + combined = reconcile_playlist(source, server) + assert combined[0]["match_status"] == "matched" + assert not combined[0]["source_track"]["image_url"] + + def test_norm_title_helper_parity(): assert norm_title("Stay (feat. X)") == "stay" assert norm_title("Song (2019 Remaster)") == "song" From 818c4f0bffb5759e607241e821004ff0abff01c9 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 11:30:58 -0700 Subject: [PATCH 009/108] =?UTF-8?q?Canonical=20album=20version=20=E2=80=94?= =?UTF-8?q?=20Stage=201:=20schema=20+=20pure=20scorer=20(dormant)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First stage of the canonical-album-version fix (#765 + #767-Bug2). Pins ONE canonical (source, album_id) per album, chosen by best-fit to the user's actual files, so the Reorganizer, Track Number Repair, and tagging stop re-resolving independently and contradicting each other. Ships DORMANT — nothing reads or writes the new data yet, so zero behavior change. Later stages populate (Stage 2) and consume (Stages 3-4) it. - core/metadata/canonical_version.py — pure scorer (the testable heart): score_release_against_files() rates a candidate release by track-count fit + duration alignment (greedy nearest within ±3s) + title overlap, dropping and renormalizing missing signals so it never crashes on sparse metadata. pick_canonical_release() takes candidates in source-priority order, picks the best fit, breaks ties toward the earlier (higher-priority) candidate so the choice is DETERMINISTIC — that determinism is what makes every tool agree (#765), while count/duration fit picks the right EDITION (#767-Bug2). A confidence floor (default 0.5) means a low-confidence guess is never pinned. - database/music_database.py — additive, nullable columns on albums (canonical_source / canonical_album_id / canonical_score / canonical_resolved_at), guarded by the existing PRAGMA-table_info pattern. NULL = unresolved = every consumer falls back to today's behavior. Tests: tests/test_canonical_version.py (11) — edition discrimination (11 files -> standard, 17 -> deluxe), deterministic priority tiebreak, duration disambiguation on count ties, graceful degradation (no durations / counts only / fuzzy titles), confidence floor, empty-input safety. tests/test_canonical_ columns_migration.py (4) — fresh DB has the columns, they're nullable w/ NULL default, migration is idempotent, and it ALTERs them onto an old albums table. 60 DB/schema regression tests still pass. --- core/metadata/canonical_version.py | 182 ++++++++++++++++++++++ database/music_database.py | 15 ++ tests/test_canonical_columns_migration.py | 69 ++++++++ tests/test_canonical_version.py | 143 +++++++++++++++++ 4 files changed, 409 insertions(+) create mode 100644 core/metadata/canonical_version.py create mode 100644 tests/test_canonical_columns_migration.py create mode 100644 tests/test_canonical_version.py diff --git a/core/metadata/canonical_version.py b/core/metadata/canonical_version.py new file mode 100644 index 00000000..0e44944e --- /dev/null +++ b/core/metadata/canonical_version.py @@ -0,0 +1,182 @@ +"""Pick the canonical album release by best-fit to the user's actual files. + +Issue #765 / #767-Bug2: SoulSync never pins ONE canonical album version per +album, so the Library Reorganizer, Track Number Repair, and tagging each +re-resolve independently and can land on different releases (standard vs +deluxe; Spotify vs MusicBrainz track numbering) and contradict each other. + +This module is the pure, testable heart of the fix: given the metadata of the +files actually on disk and a set of candidate releases, score each release by +how well it FITS those files and pick the best. "Best-fit to the files" means: + + - track-count fit — a 17-track deluxe is a poor fit for 11 files on disk + - duration alignment — each file should line up with a release track by length + - title overlap — a tiebreaker / sanity check + +What this does and does NOT solve: + - It DOES pick the right EDITION (standard vs deluxe) — the discriminating + signal is track count + durations. + - It does NOT (and cannot) decide which of two listings of the SAME album is + "more correct" when they differ only in track numbering (same files match + both equally). Instead ``pick_canonical_release`` is DETERMINISTIC and + breaks ties toward the earlier candidate — so the caller passes candidates + in source-priority order and every tool that reads the pinned result agrees + on the same release. Agreement is what resolves #765, not picking a + "winner" of the numbering disagreement. + +Pure, no I/O. Callers fetch candidate tracklists and read on-disk file metadata; +this module only scores. +""" + +from __future__ import annotations + +import re +from difflib import SequenceMatcher +from typing import Any, Dict, List, Optional, Tuple + +# Weights for the three fit signals. Count + duration dominate because "matches +# my files" is fundamentally about having the right NUMBER of the right-LENGTH +# tracks; title is a tiebreaker. Missing signals are dropped and the present +# ones renormalized (see _combine). +_W_COUNT = 0.4 +_W_DURATION = 0.4 +_W_TITLE = 0.2 + +_DEFAULT_DURATION_TOLERANCE_MS = 3000 # ±3s — covers encode/version length jitter +_DEFAULT_MIN_SCORE = 0.5 # never pin below this — leave unresolved +_TITLE_FUZZY_THRESHOLD = 0.85 + + +def _norm_title(text: str) -> str: + """Lowercase, drop bracketed qualifiers ((feat. …), [Remastered]), strip + punctuation, collapse whitespace.""" + if not text: + return "" + t = str(text).lower() + t = re.sub(r"[\(\[].*?[\)\]]", "", t) + t = re.sub(r"[^a-z0-9 ]", " ", t) + return " ".join(t.split()) + + +def _count_fit(n_files: int, n_release: int) -> float: + """1.0 when track counts match; decays with the relative difference.""" + if n_files <= 0 or n_release <= 0: + return 0.0 + return 1.0 - min(1.0, abs(n_files - n_release) / max(n_files, n_release)) + + +def _duration_fit( + file_tracks: List[Dict[str, Any]], + release_tracks: List[Dict[str, Any]], + tolerance_ms: int, +) -> Optional[float]: + """Fraction of tracks that line up by duration (greedy nearest match within + tolerance), over the larger of the two track counts — so missing or extra + tracks are penalised. Returns ``None`` when neither side has durations.""" + f_durs = [int(f["duration_ms"]) for f in file_tracks if f.get("duration_ms")] + r_durs = [int(r["duration_ms"]) for r in release_tracks if r.get("duration_ms")] + if not f_durs or not r_durs: + return None + used = [False] * len(r_durs) + matched = 0 + for fd in f_durs: + best_j, best_diff = -1, tolerance_ms + 1 + for j, rd in enumerate(r_durs): + if used[j]: + continue + diff = abs(fd - rd) + if diff <= tolerance_ms and diff < best_diff: + best_diff, best_j = diff, j + if best_j >= 0: + used[best_j] = True + matched += 1 + denom = max(len(file_tracks), len(release_tracks)) + return matched / denom if denom else 0.0 + + +def _title_fit( + file_tracks: List[Dict[str, Any]], + release_tracks: List[Dict[str, Any]], +) -> Optional[float]: + """Fraction of files whose title matches some release title (exact-normalised + or fuzzy), over the larger track count. ``None`` when titles are absent.""" + f_titles = [_norm_title(f.get("title", "")) for f in file_tracks] + f_titles = [t for t in f_titles if t] + r_titles = [_norm_title(r.get("title", "")) for r in release_tracks] + r_titles = [t for t in r_titles if t] + if not f_titles or not r_titles: + return None + r_set = set(r_titles) + matched = 0 + for ft in f_titles: + if ft in r_set or any( + SequenceMatcher(None, ft, rt).ratio() >= _TITLE_FUZZY_THRESHOLD + for rt in r_titles + ): + matched += 1 + denom = max(len(file_tracks), len(release_tracks)) + return matched / denom if denom else 0.0 + + +def _combine(parts: List[Tuple[Optional[float], float]]) -> float: + """Weighted mean over present (non-None) components, renormalising weights.""" + present = [(v, w) for v, w in parts if v is not None] + total_w = sum(w for _, w in present) + if total_w <= 0: + return 0.0 + return sum(v * w for v, w in present) / total_w + + +def score_release_against_files( + file_tracks: List[Dict[str, Any]], + release_tracks: List[Dict[str, Any]], + *, + duration_tolerance_ms: int = _DEFAULT_DURATION_TOLERANCE_MS, +) -> float: + """Score 0.0–1.0 of how well ``release_tracks`` fits the on-disk + ``file_tracks``. Each track dict may carry ``duration_ms`` and ``title``; + missing signals are dropped and the rest renormalised so the function never + crashes on sparse metadata (it just leans on what's available).""" + if not file_tracks or not release_tracks: + return 0.0 + count = _count_fit(len(file_tracks), len(release_tracks)) + dur = _duration_fit(file_tracks, release_tracks, duration_tolerance_ms) + title = _title_fit(file_tracks, release_tracks) + return _combine([(count, _W_COUNT), (dur, _W_DURATION), (title, _W_TITLE)]) + + +def pick_canonical_release( + file_tracks: List[Dict[str, Any]], + candidates: List[Dict[str, Any]], + *, + min_score: float = _DEFAULT_MIN_SCORE, + duration_tolerance_ms: int = _DEFAULT_DURATION_TOLERANCE_MS, +) -> Tuple[Optional[Dict[str, Any]], float]: + """Choose the best-fit candidate release for the on-disk files. + + ``candidates`` is a list of dicts each with a ``'tracks'`` list (plus any + caller fields like ``source``/``album_id``, returned untouched). **Pass + candidates in source-priority order** — ties break toward the EARLIER one, + so the choice is deterministic and priority-respecting (this is what makes + every tool agree, #765). + + Returns ``(best_candidate, score)``, or ``(None, best_score)`` when nothing + clears ``min_score`` — so a low-confidence guess is never pinned (the caller + leaves the album unresolved and falls back to today's behaviour).""" + best: Optional[Dict[str, Any]] = None + best_score = 0.0 + for cand in candidates: + score = score_release_against_files( + file_tracks, cand.get("tracks") or [], + duration_tolerance_ms=duration_tolerance_ms, + ) + # Strictly-greater so equal scores keep the earlier (higher-priority) + # candidate — deterministic tiebreak. + if score > best_score + 1e-9: + best, best_score = cand, score + if best is None or best_score < min_score: + return None, best_score + return best, best_score + + +__all__ = ["score_release_against_files", "pick_canonical_release"] diff --git a/database/music_database.py b/database/music_database.py index 08d42d14..61847317 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -956,6 +956,21 @@ class MusicDatabase: if album_cols and 'api_track_count' not in album_cols: cursor.execute("ALTER TABLE albums ADD COLUMN api_track_count INTEGER DEFAULT NULL") logger.info("Repaired missing api_track_count column on albums table") + + # Canonical album version (#765 / #767-Bug2). Additive + nullable: + # a NULL canonical means "unresolved" and every tool falls back to + # today's behavior, so this is safe to ship dormant. Columns are + # populated/consumed in later stages. + _canonical_cols = { + 'canonical_source': 'TEXT DEFAULT NULL', + 'canonical_album_id': 'TEXT DEFAULT NULL', + 'canonical_score': 'REAL DEFAULT NULL', + 'canonical_resolved_at': 'TIMESTAMP DEFAULT NULL', + } + for _col, _typedef in _canonical_cols.items(): + if album_cols and _col not in album_cols: + cursor.execute(f"ALTER TABLE albums ADD COLUMN {_col} {_typedef}") + logger.info("Added %s column to albums table (canonical version)", _col) except Exception as e: logger.error("Error repairing core media schema columns: %s", e) diff --git a/tests/test_canonical_columns_migration.py b/tests/test_canonical_columns_migration.py new file mode 100644 index 00000000..0531bbf1 --- /dev/null +++ b/tests/test_canonical_columns_migration.py @@ -0,0 +1,69 @@ +"""Migration test for the canonical-album-version columns (#765 Stage 1). + +Additive + nullable, so it must: appear on a fresh DB, be idempotent (re-running +the migration is a no-op, not an error), and ALTER them onto an older albums +table that lacks them. NULL = unresolved = tools fall back to today's behavior. +""" + +from __future__ import annotations + +import sqlite3 + +from database.music_database import MusicDatabase + +_CANONICAL_COLS = { + 'canonical_source', 'canonical_album_id', 'canonical_score', 'canonical_resolved_at', +} + + +def _album_cols(cur): + cur.execute("PRAGMA table_info(albums)") + return {c[1] for c in cur.fetchall()} + + +def test_fresh_db_has_canonical_columns(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + cur = db._get_connection().cursor() + assert _CANONICAL_COLS <= _album_cols(cur) + + +def test_canonical_columns_default_null(tmp_path): + # Unresolved by default -> every consumer falls back. Verify each canonical + # column declares DEFAULT NULL and is nullable (notnull flag == 0). + db = MusicDatabase(str(tmp_path / "m.db")) + cur = db._get_connection().cursor() + cur.execute("PRAGMA table_info(albums)") + info = {c[1]: c for c in cur.fetchall()} # name -> (cid, name, type, notnull, dflt, pk) + for col in _CANONICAL_COLS: + assert col in info, f"{col} missing" + assert info[col][3] == 0, f"{col} must be nullable" + dflt = info[col][4] + assert dflt is None or str(dflt).upper() == 'NULL', f"{col} default should be NULL" + + +def test_migration_is_idempotent(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + cur = db._get_connection().cursor() + before = _album_cols(cur) + # Re-running must not raise (the PRAGMA guard skips existing columns). + db._ensure_core_media_schema_columns(cur) + db._ensure_core_media_schema_columns(cur) + assert _album_cols(cur) == before + assert _CANONICAL_COLS <= _album_cols(cur) + + +def test_migration_adds_columns_to_old_albums_table(tmp_path): + # Simulate an upgraded DB whose albums table predates these columns. + path = str(tmp_path / "old.db") + conn = sqlite3.connect(path) + conn.execute("CREATE TABLE albums (id INTEGER PRIMARY KEY, title TEXT)") + conn.commit() + cur = conn.cursor() + assert not (_CANONICAL_COLS & _album_cols(cur)) # none present yet + + # Run the real migration against this old cursor. + db = MusicDatabase(str(tmp_path / "scratch.db")) + db._ensure_core_media_schema_columns(cur) + conn.commit() + + assert _CANONICAL_COLS <= _album_cols(cur) diff --git a/tests/test_canonical_version.py b/tests/test_canonical_version.py new file mode 100644 index 00000000..bb8b8558 --- /dev/null +++ b/tests/test_canonical_version.py @@ -0,0 +1,143 @@ +"""Extreme battery for canonical-album-version scoring (#765 / #767-Bug2). + +The scorer must: pick the right EDITION by best-fit to the files on disk +(standard when you have the standard, deluxe when you have the deluxe), break +ties deterministically toward the higher-priority candidate (so every tool +agrees), degrade gracefully when durations/titles are missing, and never pin a +low-confidence guess. +""" + +from __future__ import annotations + +from core.metadata.canonical_version import ( + pick_canonical_release, + score_release_against_files, +) + + +# Helpers — build track lists ---------------------------------------------- + +def _tracks(n, base_ms=180_000, step_ms=10_000, titles=None): + """n tracks with distinct, deterministic durations + optional titles.""" + out = [] + for i in range(n): + t = {"duration_ms": base_ms + i * step_ms, "track_number": i + 1} + if titles: + t["title"] = titles[i] + out.append(t) + return out + + +STANDARD_TITLES = [f"Song {i+1}" for i in range(11)] +DELUXE_TITLES = STANDARD_TITLES + [f"Bonus {i+1}" for i in range(6)] + + +# ── edition discrimination ──────────────────────────────────────────────── + +def test_eleven_files_prefer_standard_over_deluxe(): + files = _tracks(11, titles=STANDARD_TITLES) + standard = _tracks(11, titles=STANDARD_TITLES) + deluxe = _tracks(17, titles=DELUXE_TITLES) + s_std = score_release_against_files(files, standard) + s_dlx = score_release_against_files(files, deluxe) + assert s_std > s_dlx + best, score = pick_canonical_release( + files, + [{"source": "standard", "tracks": standard}, {"source": "deluxe", "tracks": deluxe}], + ) + assert best["source"] == "standard" and score > 0.9 + + +def test_seventeen_files_prefer_deluxe(): + files = _tracks(17, titles=DELUXE_TITLES) + standard = _tracks(11, titles=STANDARD_TITLES) + deluxe = _tracks(17, titles=DELUXE_TITLES) + best, _ = pick_canonical_release( + files, + # deluxe deliberately listed SECOND to prove count/fit beats order here + [{"source": "standard", "tracks": standard}, {"source": "deluxe", "tracks": deluxe}], + ) + assert best["source"] == "deluxe" + + +def test_exact_count_and_durations_scores_near_one(): + files = _tracks(11, titles=STANDARD_TITLES) + assert score_release_against_files(files, _tracks(11, titles=STANDARD_TITLES)) > 0.99 + + +# ── deterministic tiebreak (the #765 resolution) ────────────────────────── + +def test_identical_releases_break_tie_to_first_candidate(): + # Same album from two sources (same files match both equally) — must pick + # the FIRST (higher-priority) deterministically so both tools agree. + files = _tracks(11, titles=STANDARD_TITLES) + a = {"source": "spotify", "tracks": _tracks(11, titles=STANDARD_TITLES)} + b = {"source": "musicbrainz", "tracks": _tracks(11, titles=STANDARD_TITLES)} + best, _ = pick_canonical_release(files, [a, b]) + assert best["source"] == "spotify" + # ...and stable when the order flips (priority is the caller's order). + best2, _ = pick_canonical_release(files, [b, a]) + assert best2["source"] == "musicbrainz" + + +# ── duration disambiguation when counts tie ─────────────────────────────── + +def test_duration_breaks_tie_when_counts_equal(): + # Two 11-track candidates; the files' durations match candidate A's lengths, + # not B's (e.g. album cuts vs radio edits). A must win on duration fit. + files = _tracks(11, base_ms=200_000, step_ms=5_000) + cand_a = {"source": "album", "tracks": _tracks(11, base_ms=200_000, step_ms=5_000)} + cand_b = {"source": "edits", "tracks": _tracks(11, base_ms=140_000, step_ms=5_000)} + best, _ = pick_canonical_release(files, [cand_b, cand_a]) # B listed first + assert best["source"] == "album" # duration fit overrides order + + +# ── graceful degradation ────────────────────────────────────────────────── + +def test_no_durations_falls_back_to_count_and_title(): + files = [{"title": t} for t in STANDARD_TITLES] # no durations + standard = [{"title": t} for t in STANDARD_TITLES] + deluxe = [{"title": t} for t in DELUXE_TITLES] + best, score = pick_canonical_release( + files, + [{"source": "standard", "tracks": standard}, {"source": "deluxe", "tracks": deluxe}], + ) + assert best["source"] == "standard" and score > 0.5 + + +def test_only_counts_available_still_scores(): + files = [{} for _ in range(11)] + assert score_release_against_files(files, [{} for _ in range(11)]) > 0.99 + assert score_release_against_files(files, [{} for _ in range(17)]) < 0.8 + + +def test_fuzzy_titles_still_match(): + files = _tracks(3, titles=["Believer", "Whatever It Takes", "Thunder"]) + rel = _tracks(3, titles=["Believer (Remastered)", "Whatever It Takes", "Thunder!"]) + assert score_release_against_files(files, rel) > 0.9 + + +# ── confidence floor / guards ───────────────────────────────────────────── + +def test_below_floor_returns_none(): + files = _tracks(11, titles=STANDARD_TITLES) + # A wildly wrong candidate (3 unrelated tracks) must not be pinned. + bad = {"source": "wrong", "tracks": _tracks(3, base_ms=60_000, titles=["X", "Y", "Z"])} + best, score = pick_canonical_release(files, [bad]) + assert best is None + assert score < 0.5 + + +def test_empty_inputs_are_safe(): + assert score_release_against_files([], _tracks(11)) == 0.0 + assert score_release_against_files(_tracks(11), []) == 0.0 + best, score = pick_canonical_release(_tracks(11), []) + assert best is None and score == 0.0 + + +def test_min_score_is_tunable(): + files = _tracks(11, titles=STANDARD_TITLES) + near = {"source": "near", "tracks": _tracks(10, titles=STANDARD_TITLES[:10])} + # default floor accepts a 10/11 fit, a strict floor rejects it + assert pick_canonical_release(files, [near])[0] is not None + assert pick_canonical_release(files, [near], min_score=0.99)[0] is None From f37bc34082b8643bebe212cd500896980284484d Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 11:36:19 -0700 Subject: [PATCH 010/108] =?UTF-8?q?Canonical=20album=20version=20=E2=80=94?= =?UTF-8?q?=20Stage=202=20(core):=20resolver=20+=20persistence=20(dormant)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turns the Stage-1 scorer into an end-to-end resolver + persists the result. Still DORMANT — no consumer reads it yet, so zero behavior change. - core/metadata/canonical_resolver.py — resolve_canonical_for_album(): builds candidate releases from the album's per-source IDs (in source-priority order), fetches each tracklist via an INJECTED fetch_tracklist (so it's unit-testable without live APIs), scores them with pick_canonical_release, and returns the best-fit {source, album_id, score}. Skips sources with no id / failed fetch; returns None when there are no files, no candidates, or nothing clears the confidence floor. - database/music_database.py — set_album_canonical() / get_album_canonical() write/read the Stage-1 columns. get returns None when unresolved, which every consumer will treat as "fall back to today's behavior". Tests: tests/test_canonical_resolver.py (7) — best-fit beats priority, priority breaks true ties, skips missing-id/failed-fetch sources, None on no-candidates/no-files/below-floor, score rounding. tests/test_canonical_db.py (4) — set/get round-trip incl. timestamp, unresolved -> None, overwrite, missing-album -> False. 34 canonical + DB-migration tests pass. Remaining for Stage 2 (the trigger): read on-disk file durations/titles for an album, gather its source IDs, call the resolver, store — wired via a backfill repair job + an enrichment hook. Then Stages 3-4 wire the Reorganizer and Track Number Repair to READ the pinned canonical. --- core/metadata/canonical_resolver.py | 76 ++++++++++++++++++++ database/music_database.py | 47 ++++++++++++ tests/test_canonical_db.py | 54 ++++++++++++++ tests/test_canonical_resolver.py | 108 ++++++++++++++++++++++++++++ 4 files changed, 285 insertions(+) create mode 100644 core/metadata/canonical_resolver.py create mode 100644 tests/test_canonical_db.py create mode 100644 tests/test_canonical_resolver.py diff --git a/core/metadata/canonical_resolver.py b/core/metadata/canonical_resolver.py new file mode 100644 index 00000000..b671a759 --- /dev/null +++ b/core/metadata/canonical_resolver.py @@ -0,0 +1,76 @@ +"""Resolve (and persist) the canonical release for an album — Stage 2 of #765. + +Stage 1 gave us the pure scorer (``core.metadata.canonical_version``). This +module turns it into an end-to-end resolver: gather the album's candidate +releases (one per metadata-source ID it has), score each against the on-disk +files, and return the best fit. Wiring (backfill job / enrichment hook) and the +DB store live alongside; the decision logic here is kept dependency-injected +(``fetch_tracklist`` is passed in) so it's fully unit-testable without live APIs +or real files. + +Still NO consumer reads the result in Stage 2 — populating the columns is +behavior-neutral. Stages 3-4 wire the Reorganizer and Track Number Repair to +read it. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, List, Optional + +from core.metadata.canonical_version import pick_canonical_release + + +def resolve_canonical_for_album( + *, + album_source_ids: Dict[str, str], + file_tracks: List[Dict[str, Any]], + fetch_tracklist: Callable[[str, str], Optional[List[Dict[str, Any]]]], + source_priority: List[str], + min_score: float = 0.5, +) -> Optional[Dict[str, Any]]: + """Pick the canonical release for one album. + + ``album_source_ids``: ``{source: album_id}`` the album is linked to. + ``file_tracks``: on-disk track metadata (``{duration_ms, title}``). + ``fetch_tracklist(source, album_id)``: returns that release's tracklist (or + None/[] on miss); injected so callers supply ``get_album_tracks_for_source`` + while tests supply a fake. + ``source_priority``: order to build candidates in — ties break toward the + earlier (higher-priority) source, keeping the choice deterministic. + + Returns ``{'source', 'album_id', 'score'}`` for the best fit, or ``None`` + when there are no files, no resolvable candidates, or nothing clears + ``min_score`` (caller leaves the album unresolved → tools fall back).""" + if not file_tracks: + return None + + candidates: List[Dict[str, Any]] = [] + for source in source_priority: + album_id = album_source_ids.get(source) + if not album_id: + continue + try: + tracks = fetch_tracklist(source, str(album_id)) + except Exception: + tracks = None + if tracks: + candidates.append({ + 'source': source, + 'album_id': str(album_id), + 'tracks': tracks, + }) + + if not candidates: + return None + + best, score = pick_canonical_release(file_tracks, candidates, min_score=min_score) + if not best: + return None + return { + 'source': best['source'], + 'album_id': best['album_id'], + 'score': round(score, 4), + } + + +__all__ = ["resolve_canonical_for_album"] diff --git a/database/music_database.py b/database/music_database.py index 61847317..74fa5e13 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -974,6 +974,53 @@ class MusicDatabase: except Exception as e: logger.error("Error repairing core media schema columns: %s", e) + def set_album_canonical(self, album_id, source: str, canonical_album_id: str, score: float) -> bool: + """Persist the resolved canonical (source, album_id, score) for an album + (#765 Stage 2). Returns True if a row was updated.""" + conn = self._get_connection() + try: + cursor = conn.cursor() + cursor.execute( + "UPDATE albums SET canonical_source = ?, canonical_album_id = ?, " + "canonical_score = ?, canonical_resolved_at = CURRENT_TIMESTAMP " + "WHERE id = ?", + (source, str(canonical_album_id), float(score), album_id), + ) + conn.commit() + return cursor.rowcount > 0 + except Exception as e: + logger.error("Error setting album canonical for %s: %s", album_id, e) + return False + finally: + conn.close() + + def get_album_canonical(self, album_id) -> Optional[dict]: + """Return ``{'source','album_id','score','resolved_at'}`` for an album's + pinned canonical release, or ``None`` when unresolved (#765 Stage 2). + Consumers treat ``None`` as 'fall back to today's behavior'.""" + conn = self._get_connection() + try: + cursor = conn.cursor() + cursor.execute( + "SELECT canonical_source, canonical_album_id, canonical_score, " + "canonical_resolved_at FROM albums WHERE id = ?", + (album_id,), + ) + row = cursor.fetchone() + if not row or not row[0] or not row[1]: + return None + return { + 'source': row[0], + 'album_id': row[1], + 'score': row[2], + 'resolved_at': row[3], + } + except Exception as e: + logger.error("Error reading album canonical for %s: %s", album_id, e) + return None + finally: + conn.close() + def _add_mirrored_playlist_explored_column(self, cursor): """Add explored_at column to mirrored_playlists to persist explore badge.""" try: diff --git a/tests/test_canonical_db.py b/tests/test_canonical_db.py new file mode 100644 index 00000000..4bc0496f --- /dev/null +++ b/tests/test_canonical_db.py @@ -0,0 +1,54 @@ +"""DB persistence for canonical album version (#765 Stage 2).""" + +from __future__ import annotations + +from database.music_database import MusicDatabase + + +def _album(db, album_id="alb_evolve"): + # id columns are TEXT (GUID) post-migration, so insert explicit ids and a + # valid FK rather than relying on integer rowids. + conn = db._get_connection() + cur = conn.cursor() + cur.execute("INSERT INTO artists (id, name) VALUES ('art_id', 'Imagine Dragons')") + cur.execute( + "INSERT INTO albums (id, title, artist_id) VALUES (?, 'Evolve', 'art_id')", + (album_id,), + ) + conn.commit() + conn.close() + return album_id + + +def test_set_then_get_roundtrip(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + album_id = _album(db) + + assert db.get_album_canonical(album_id) is None # unresolved by default + + assert db.set_album_canonical(album_id, "spotify", "sp_evolve_123", 0.97) is True + got = db.get_album_canonical(album_id) + assert got["source"] == "spotify" + assert got["album_id"] == "sp_evolve_123" + assert abs(got["score"] - 0.97) < 1e-6 + assert got["resolved_at"] # timestamp populated + + +def test_get_unresolved_returns_none(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + album_id = _album(db) + assert db.get_album_canonical(album_id) is None + + +def test_set_overwrites_previous(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + album_id = _album(db) + db.set_album_canonical(album_id, "spotify", "old", 0.6) + db.set_album_canonical(album_id, "musicbrainz", "new", 0.95) + got = db.get_album_canonical(album_id) + assert got["source"] == "musicbrainz" and got["album_id"] == "new" + + +def test_set_on_missing_album_returns_false(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + assert db.set_album_canonical(999999, "spotify", "x", 0.9) is False diff --git a/tests/test_canonical_resolver.py b/tests/test_canonical_resolver.py new file mode 100644 index 00000000..2868a373 --- /dev/null +++ b/tests/test_canonical_resolver.py @@ -0,0 +1,108 @@ +"""Tests for resolve_canonical_for_album (#765 Stage 2 — injectable core).""" + +from __future__ import annotations + +from core.metadata.canonical_resolver import resolve_canonical_for_album + +STD = [{"duration_ms": 180_000 + i * 10_000, "title": f"Song {i+1}"} for i in range(11)] +DLX = STD + [{"duration_ms": 300_000 + i * 10_000, "title": f"Bonus {i+1}"} for i in range(6)] + +PRIORITY = ["spotify", "itunes", "deezer", "musicbrainz"] + + +def _fetcher(table): + """fetch_tracklist backed by a {(source, album_id): tracks} table.""" + def fetch(source, album_id): + return table.get((source, album_id)) + return fetch + + +def test_picks_source_whose_release_fits_the_files(): + files = list(STD) # user owns the 11-track standard + table = { + ("spotify", "sp_deluxe"): DLX, # spotify linked to deluxe (17) + ("musicbrainz", "mb_std"): STD, # musicbrainz has standard (11) + } + out = resolve_canonical_for_album( + album_source_ids={"spotify": "sp_deluxe", "musicbrainz": "mb_std"}, + file_tracks=files, + fetch_tracklist=_fetcher(table), + source_priority=PRIORITY, + ) + # Best FIT wins over priority — standard matches the files, deluxe doesn't. + assert out == {"source": "musicbrainz", "album_id": "mb_std", "score": out["score"]} + assert out["score"] > 0.9 + + +def test_priority_breaks_tie_between_equal_fits(): + files = list(STD) + table = {("spotify", "a"): STD, ("itunes", "b"): STD} # identical fit + out = resolve_canonical_for_album( + album_source_ids={"itunes": "b", "spotify": "a"}, + file_tracks=files, + fetch_tracklist=_fetcher(table), + source_priority=PRIORITY, # spotify before itunes + ) + assert out["source"] == "spotify" + + +def test_skips_sources_without_ids_or_failed_fetch(): + files = list(STD) + + def fetch(source, album_id): + if source == "spotify": + raise RuntimeError("API down") + if source == "deezer": + return STD + return None + + out = resolve_canonical_for_album( + album_source_ids={"spotify": "x", "deezer": "y"}, # no itunes id + file_tracks=files, + fetch_tracklist=fetch, + source_priority=PRIORITY, + ) + assert out["source"] == "deezer" + + +def test_none_when_no_candidates(): + out = resolve_canonical_for_album( + album_source_ids={}, + file_tracks=list(STD), + fetch_tracklist=_fetcher({}), + source_priority=PRIORITY, + ) + assert out is None + + +def test_none_when_no_files(): + out = resolve_canonical_for_album( + album_source_ids={"spotify": "a"}, + file_tracks=[], + fetch_tracklist=_fetcher({("spotify", "a"): STD}), + source_priority=PRIORITY, + ) + assert out is None + + +def test_none_when_below_floor(): + files = list(STD) # 11 tracks + # Only candidate is a wildly-wrong 3-track release. + table = {("spotify", "a"): [{"duration_ms": 60_000, "title": "X"}] * 3} + out = resolve_canonical_for_album( + album_source_ids={"spotify": "a"}, + file_tracks=files, + fetch_tracklist=_fetcher(table), + source_priority=PRIORITY, + ) + assert out is None + + +def test_score_is_rounded(): + out = resolve_canonical_for_album( + album_source_ids={"spotify": "a"}, + file_tracks=list(STD), + fetch_tracklist=_fetcher({("spotify", "a"): STD}), + source_priority=PRIORITY, + ) + assert out["score"] == round(out["score"], 4) From 43878b4d3da5e0692a556877b383d733abf7ead9 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 11:42:20 -0700 Subject: [PATCH 011/108] =?UTF-8?q?Canonical=20album=20version=20=E2=80=94?= =?UTF-8?q?=20Stage=202=20(trigger):=20resolve+store=20orchestration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes Stage 2's populate path. Still dormant — no consumer calls it yet. - resolve_and_store_canonical_for_album(db, album_id, ...): loads the album's source IDs + its tracks' (duration_ms, title) from the DB via the SAME loader the Reorganizer uses (load_album_and_tracks + _extract_source_ids), so the canonical is chosen over exactly the source IDs the reorganizer sees; scores off the DB track rows (the library's view of the files — no per-file disk reads), resolves the best fit, and persists it. Returns the stored result or None when unresolved. - default_fetch_tracklist(): production fetcher wrapping get_album_tracks_for_source, normalising to {title, track_number, duration_ms} (duration best-effort; sec->ms; absent -> scorer leans on count+title). Design note: chose LAZY resolution (Stages 3-4 consumers call this when they hit an album with no canonical) over a standalone backfill repair job — no new scheduling/UI surface, resolves only when a tool actually needs it, and stays gated (NULL canonical = today's behavior). Tests: tests/test_canonical_orchestration.py (5) — end-to-end on a real temp DB (11 files pick the 11-track release over a 17-track deluxe and persist it), no-source-ids -> None, missing-album -> None, and default_fetch_tracklist normalization (dict items, seconds->ms) + failure -> None. All canonical + DB-migration tests green. --- core/metadata/canonical_resolver.py | 88 +++++++++++++++++++++++- tests/test_canonical_orchestration.py | 97 +++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 tests/test_canonical_orchestration.py diff --git a/core/metadata/canonical_resolver.py b/core/metadata/canonical_resolver.py index b671a759..a240870a 100644 --- a/core/metadata/canonical_resolver.py +++ b/core/metadata/canonical_resolver.py @@ -73,4 +73,90 @@ def resolve_canonical_for_album( } -__all__ = ["resolve_canonical_for_album"] +def default_fetch_tracklist(source: str, album_id: str) -> Optional[List[Dict[str, Any]]]: + """Production ``fetch_tracklist``: pull a release's tracklist from a metadata + source and normalise to ``{title, track_number, duration_ms}``. Duration is + best-effort (not every source exposes it); when absent the scorer just leans + on track-count + title. Returns None on any failure.""" + try: + from core.metadata_service import get_album_tracks_for_source + data = get_album_tracks_for_source(source, album_id) + except Exception: + return None + items = data if isinstance(data, list) else ( + (data.get('items') or data.get('tracks') or []) if isinstance(data, dict) else [] + ) + if isinstance(items, dict): # {'tracks': {'items': [...]}} + items = items.get('items') or [] + out: List[Dict[str, Any]] = [] + for it in items: + get = it.get if isinstance(it, dict) else (lambda k, d=None: getattr(it, k, d)) + dur = get('duration_ms') + if dur is None: + secs = get('duration') # some sources give seconds + dur = int(secs * 1000) if isinstance(secs, (int, float)) and secs else None + out.append({ + 'title': get('name') or get('title') or '', + 'track_number': get('track_number'), + 'duration_ms': dur, + }) + return out or None + + +def resolve_and_store_canonical_for_album( + db, + album_id, + *, + fetch_tracklist: Optional[Callable[[str, str], Any]] = None, + source_priority: Optional[List[str]] = None, + min_score: float = 0.5, +) -> Optional[Dict[str, Any]]: + """Gather an album's source IDs + its tracks' (duration, title) from the DB, + resolve the best-fit canonical release, and persist it. Returns the stored + ``{source, album_id, score}`` or None when unresolved. + + Uses the SAME album/source-id loader the Reorganizer uses + (``load_album_and_tracks`` + ``_extract_source_ids``) so the canonical is + chosen over exactly the source IDs the reorganizer sees. Scores off the DB + track rows' ``duration`` (stored in ms) + ``title`` — the library's view of + the files — so no per-file disk reads are needed.""" + from core.library_reorganize import _extract_source_ids, load_album_and_tracks + + album_data, tracks = load_album_and_tracks(db, album_id) + if not album_data or not tracks: + return None + source_ids = {s: v for s, v in _extract_source_ids(album_data).items() if v} + if not source_ids: + return None + + file_tracks = [ + {'duration_ms': t.get('duration') or 0, 'title': t.get('title') or ''} + for t in tracks + ] + + if fetch_tracklist is None: + fetch_tracklist = default_fetch_tracklist + if source_priority is None: + try: + from core.metadata_service import get_primary_source, get_source_priority + source_priority = get_source_priority(get_primary_source()) + except Exception: + source_priority = list(source_ids.keys()) + + result = resolve_canonical_for_album( + album_source_ids=source_ids, + file_tracks=file_tracks, + fetch_tracklist=fetch_tracklist, + source_priority=source_priority, + min_score=min_score, + ) + if result: + db.set_album_canonical(album_id, result['source'], result['album_id'], result['score']) + return result + + +__all__ = [ + "resolve_canonical_for_album", + "resolve_and_store_canonical_for_album", + "default_fetch_tracklist", +] diff --git a/tests/test_canonical_orchestration.py b/tests/test_canonical_orchestration.py new file mode 100644 index 00000000..2f3ae8a0 --- /dev/null +++ b/tests/test_canonical_orchestration.py @@ -0,0 +1,97 @@ +"""End-to-end orchestration for canonical resolve+store (#765 Stage 2 trigger). + +Uses a real temp DB (album + tracks + source IDs) and an INJECTED fetcher, so +the DB gathering + persistence are exercised for real without live APIs. +""" + +from __future__ import annotations + +from core.metadata.canonical_resolver import ( + default_fetch_tracklist, + resolve_and_store_canonical_for_album, +) +from database.music_database import MusicDatabase + +STD = [{"duration_ms": 180_000 + i * 10_000, "title": f"Song {i+1}", "track_number": i + 1} for i in range(11)] +DLX = STD + [{"duration_ms": 320_000 + i * 10_000, "title": f"Bonus {i+1}", "track_number": 12 + i} for i in range(6)] + + +def _seed(db, *, spotify=None, deezer=None, n_files=11): + """Insert an album (with given source IDs) + n_files tracks whose + durations/titles match the STANDARD release.""" + conn = db._get_connection() + cur = conn.cursor() + cur.execute("INSERT INTO artists (id, name) VALUES ('art1', 'Imagine Dragons')") + cur.execute( + "INSERT INTO albums (id, title, artist_id, spotify_album_id, deezer_id) " + "VALUES ('alb1', 'Evolve', 'art1', ?, ?)", + (spotify, deezer), + ) + for i in range(n_files): + cur.execute( + "INSERT INTO tracks (id, album_id, artist_id, title, track_number, duration) " + "VALUES (?, 'alb1', 'art1', ?, ?, ?)", + (f"t{i}", f"Song {i+1}", i + 1, 180_000 + i * 10_000), + ) + conn.commit() + conn.close() + return "alb1" + + +def test_resolve_and_store_picks_best_fit_and_persists(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + album_id = _seed(db, spotify="sp_deluxe", deezer="dz_std") # 11 files + + table = {("spotify", "sp_deluxe"): DLX, ("deezer", "dz_std"): STD} + out = resolve_and_store_canonical_for_album( + db, album_id, + fetch_tracklist=lambda s, a: table.get((s, a)), + source_priority=["spotify", "deezer"], + ) + # Deezer's standard matches the 11 files better than Spotify's deluxe. + assert out["source"] == "deezer" and out["album_id"] == "dz_std" + # ...and it was persisted. + stored = db.get_album_canonical(album_id) + assert stored["source"] == "deezer" and stored["album_id"] == "dz_std" + + +def test_resolve_returns_none_when_album_has_no_source_ids(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + album_id = _seed(db, spotify=None, deezer=None) + out = resolve_and_store_canonical_for_album( + db, album_id, fetch_tracklist=lambda s, a: STD, source_priority=["spotify"], + ) + assert out is None + assert db.get_album_canonical(album_id) is None + + +def test_resolve_returns_none_for_missing_album(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + out = resolve_and_store_canonical_for_album( + db, "does-not-exist", fetch_tracklist=lambda s, a: STD, source_priority=["spotify"], + ) + assert out is None + + +# ── default_fetch_tracklist normalization (no DB / no live API) ──────────── + +def test_default_fetcher_normalizes_dict_items(monkeypatch): + import core.metadata_service as ms + monkeypatch.setattr( + ms, "get_album_tracks_for_source", + lambda s, a: [{"name": "A", "track_number": 1, "duration_ms": 200000}, + {"title": "B", "track_number": 2, "duration": 210}], # seconds + raising=False, + ) + out = default_fetch_tracklist("spotify", "x") + assert out[0] == {"title": "A", "track_number": 1, "duration_ms": 200000} + assert out[1] == {"title": "B", "track_number": 2, "duration_ms": 210_000} # sec->ms + + +def test_default_fetcher_handles_failure(monkeypatch): + import core.metadata_service as ms + monkeypatch.setattr( + ms, "get_album_tracks_for_source", + lambda s, a: (_ for _ in ()).throw(RuntimeError("boom")), raising=False, + ) + assert default_fetch_tracklist("spotify", "x") is None From ecdfde03c67a65b5cf5c6afff246c8691c8523f4 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 11:45:31 -0700 Subject: [PATCH 012/108] =?UTF-8?q?Canonical=20album=20version=20=E2=80=94?= =?UTF-8?q?=20Stage=203:=20Reorganizer=20prefers=20pinned=20canonical=20(r?= =?UTF-8?q?ead)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _resolve_source now prefers the album's pinned canonical (source, album_id) when set, before the source-priority walk. So once an album's canonical is resolved, reorganize agrees with Track Number Repair (Stage 4) and stops mislabelling a standard album as deluxe (#767-Bug2). Gated + side-effect-free: only changes behavior for albums that already carry a canonical (none do until the populate step runs), an explicit user source pick (strict_source) still wins over the canonical, and a failed canonical fetch falls through to today's priority walk. So this stage is behavior-neutral until canonical is populated. Tests: tests/test_reorganize_canonical_source.py (4) — canonical preferred over priority, fetch-failure falls back, strict_source ignores canonical, no-canonical unchanged. 113 reorganize-orchestrator/tag-source/unknown-artist tests still pass (no regression). --- core/library_reorganize.py | 19 ++++++ tests/test_reorganize_canonical_source.py | 75 +++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 tests/test_reorganize_canonical_source.py diff --git a/core/library_reorganize.py b/core/library_reorganize.py index 25adcda4..02509c11 100644 --- a/core/library_reorganize.py +++ b/core/library_reorganize.py @@ -227,6 +227,25 @@ def _resolve_source(album_data: dict, primary_source: str, strict_source: bool = """ source_ids = _extract_source_ids(album_data) + # #765: if a canonical release was pinned for this album (best-fit to the + # user's actual files), prefer it — so reorganize agrees with Track Number + # Repair and stops mislabelling standard albums as deluxe (#767-Bug2). Gated + # on the album row carrying a canonical, and skipped when the user explicitly + # picked a source in the modal (strict_source) — their choice wins. Falls + # through to the priority walk if the canonical fetch fails. + if not strict_source: + c_source = album_data.get('canonical_source') + c_id = album_data.get('canonical_album_id') + if c_source and c_id: + try: + api_album = get_album_for_source(c_source, c_id) + api_tracks = get_album_tracks_for_source(c_source, c_id) + items = _normalize_album_tracks(api_tracks) + if items and api_album: + return c_source, api_album, items + except Exception as e: + logger.warning(f"[Reorganize] canonical {c_source} lookup raised: {e}") + if strict_source: sources_to_try = [primary_source] if primary_source else [] else: diff --git a/tests/test_reorganize_canonical_source.py b/tests/test_reorganize_canonical_source.py new file mode 100644 index 00000000..711638ed --- /dev/null +++ b/tests/test_reorganize_canonical_source.py @@ -0,0 +1,75 @@ +"""_resolve_source honors a pinned canonical release (#765 Stage 3, read side). + +Gated + side-effect-free: only changes behavior for albums that already carry a +canonical_source/canonical_album_id, and an explicit user source pick +(strict_source) still wins. No canonical -> byte-identical to before. +""" + +from __future__ import annotations + +import core.library_reorganize as lr + + +def _patch_fetch(monkeypatch, tracklists): + """tracklists: {(source, album_id): items_or_None}. Patches the album + + tracklist fetchers and the normaliser (pass-through).""" + def get_album(source, aid): + return {"name": f"{source}:{aid}"} if tracklists.get((source, aid)) else None + + def get_tracks(source, aid): + return tracklists.get((source, aid)) + + monkeypatch.setattr(lr, "get_album_for_source", get_album) + monkeypatch.setattr(lr, "get_album_tracks_for_source", get_tracks) + monkeypatch.setattr(lr, "_normalize_album_tracks", lambda items: items or []) + monkeypatch.setattr(lr, "get_source_priority", lambda primary: ["spotify", "itunes", "deezer"]) + + +def test_canonical_source_preferred_over_priority(monkeypatch): + # Album has spotify (priority winner) AND a pinned canonical = deezer. + _patch_fetch(monkeypatch, { + ("spotify", "sp1"): [{"name": "x"}], + ("deezer", "dz1"): [{"name": "y"}], + }) + album_data = { + "spotify_album_id": "sp1", "deezer_id": "dz1", + "canonical_source": "deezer", "canonical_album_id": "dz1", + } + source, api_album, items = lr._resolve_source(album_data, "spotify") + assert source == "deezer" # canonical beats the priority walk + + +def test_canonical_fetch_failure_falls_back_to_priority(monkeypatch): + # Canonical points at musicbrainz but that fetch yields nothing -> fall back. + _patch_fetch(monkeypatch, { + ("spotify", "sp1"): [{"name": "x"}], + # no entry for ('musicbrainz', 'mb1') -> get_tracks returns None + }) + album_data = { + "spotify_album_id": "sp1", + "canonical_source": "musicbrainz", "canonical_album_id": "mb1", + } + source, _, _ = lr._resolve_source(album_data, "spotify") + assert source == "spotify" # fell back to priority + + +def test_strict_source_ignores_canonical(monkeypatch): + # User explicitly picked spotify in the modal — their choice wins over canonical. + _patch_fetch(monkeypatch, { + ("spotify", "sp1"): [{"name": "x"}], + ("deezer", "dz1"): [{"name": "y"}], + }) + album_data = { + "spotify_album_id": "sp1", "deezer_id": "dz1", + "canonical_source": "deezer", "canonical_album_id": "dz1", + } + source, _, _ = lr._resolve_source(album_data, "spotify", strict_source=True) + assert source == "spotify" + + +def test_no_canonical_unchanged(monkeypatch): + # No canonical set -> identical to legacy priority resolution. + _patch_fetch(monkeypatch, {("spotify", "sp1"): [{"name": "x"}]}) + album_data = {"spotify_album_id": "sp1"} + source, _, _ = lr._resolve_source(album_data, "spotify") + assert source == "spotify" From f5752e3dc056c62937b878677692e9df150ca10f Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 11:47:42 -0700 Subject: [PATCH 013/108] =?UTF-8?q?Canonical=20album=20version=20=E2=80=94?= =?UTF-8?q?=20Stage=204:=20Track=20Number=20Repair=20prefers=20canonical?= =?UTF-8?q?=20(read)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _resolve_album_tracklist gains a Fallback -1: if the album has a pinned canonical (source, album_id), use it before the existing 6-level cascade — so Track Number Repair resolves the SAME release the Reorganizer does (Stage 3) and the two stop contradicting each other (#765, the Spotify-4 vs MusicBrainz-3 conflict). Gated + additive: the entire existing cascade is untouched for albums without a canonical, so this job's all-01-album rescue (which relies on the MusicBrainz/ AudioDB fallbacks for albums with no DB source ID) is fully preserved — that's the regression we explicitly refused to take in a reactive fix. New helper _lookup_canonical_from_db() mirrors _lookup_album_ids_from_db (file-path -> track -> album), returns None when no DB / no match / columns absent / unresolved. Tests: tests/test_track_repair_canonical.py (4) — returns canonical when pinned, None when unresolved / file untracked / no DB. Existing track_number_repair tests still pass (no regression). --- core/repair_jobs/track_number_repair.py | 56 +++++++++++++++++++++++++ tests/test_track_repair_canonical.py | 52 +++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 tests/test_track_repair_canonical.py diff --git a/core/repair_jobs/track_number_repair.py b/core/repair_jobs/track_number_repair.py index 8a569de6..a7c4292a 100644 --- a/core/repair_jobs/track_number_repair.py +++ b/core/repair_jobs/track_number_repair.py @@ -275,6 +275,22 @@ class TrackNumberRepairJob(RepairJob): primary_source = get_primary_source() source_priority = get_source_priority(primary_source) + # Fallback -1 (#765): a pinned canonical release wins over the whole + # cascade below — so Track Number Repair resolves the SAME release the + # Reorganizer does (Stage 3) and the two stop contradicting each other. + # Gated on the album carrying a canonical; everything below is untouched + # for albums without one (preserving the all-01-album rescue this job + # exists for — the regression we refused to take in a reactive fix). + canonical = _lookup_canonical_from_db(file_track_data, context) + if canonical: + c_source, c_id = canonical + if _is_valid_album_id(c_id): + tracks = _get_album_tracklist(c_source, c_id, cache) + if tracks: + logger.info("[Repair] %s — resolved via canonical %s album ID: %s", + folder_name, c_source, c_id) + return tracks + # Fallback 0: Check DB first. If any tracked file already has source IDs, # prefer the configured source order and use the first available album ID. source_album_ids = _lookup_album_ids_from_db(file_track_data, context) @@ -806,6 +822,46 @@ def _update_db_file_path(db, old_path: str, new_path: str): conn.close() +def _lookup_canonical_from_db(file_track_data: List[Tuple[str, str, Any]], + context: JobContext) -> Optional[Tuple[str, str]]: + """Return the album's pinned canonical ``(source, album_id)`` or None. + + #765: when the album this folder's files belong to has a canonical release + pinned (best-fit to the files), Track Number Repair uses it first so it + agrees with the Reorganizer. Resolves by matching a file path to its DB + track row. None when no DB, no match, columns absent, or unresolved.""" + if not context.db: + return None + conn = None + try: + conn = context.db._get_connection() + cursor = conn.cursor() + cursor.execute("PRAGMA table_info(albums)") + cols = {row[1] for row in cursor.fetchall()} + if 'canonical_source' not in cols or 'canonical_album_id' not in cols: + return None + for fpath, _, _ in file_track_data: + cursor.execute( + """ + SELECT al.canonical_source, al.canonical_album_id + FROM tracks t + JOIN albums al ON al.id = t.album_id + WHERE t.file_path = ? + LIMIT 1 + """, + (fpath,), + ) + row = cursor.fetchone() + if row and row[0] and row[1]: + return (str(row[0]), str(row[1])) + except Exception as e: + logger.debug("Error looking up canonical from DB: %s", e) + finally: + if conn: + conn.close() + return None + + def _lookup_album_ids_from_db(file_track_data: List[Tuple[str, str, Any]], context: JobContext) -> Dict[str, Optional[str]]: """Look up album IDs from the database using file paths. diff --git a/tests/test_track_repair_canonical.py b/tests/test_track_repair_canonical.py new file mode 100644 index 00000000..72879151 --- /dev/null +++ b/tests/test_track_repair_canonical.py @@ -0,0 +1,52 @@ +"""Track Number Repair canonical lookup (#765 Stage 4, read side).""" + +from __future__ import annotations + +import types + +from core.repair_jobs.track_number_repair import _lookup_canonical_from_db +from database.music_database import MusicDatabase + + +def _ctx(db): + return types.SimpleNamespace(db=db) + + +def _seed(db, *, with_canonical: bool, file_path: str = "/music/Evolve/01 - Believer.flac"): + conn = db._get_connection() + cur = conn.cursor() + cur.execute("INSERT INTO artists (id, name) VALUES ('art1', 'Imagine Dragons')") + cur.execute("INSERT INTO albums (id, title, artist_id) VALUES ('alb1', 'Evolve', 'art1')") + cur.execute( + "INSERT INTO tracks (id, album_id, artist_id, title, track_number, duration, file_path) " + "VALUES ('t1', 'alb1', 'art1', 'Believer', 1, 204000, ?)", + (file_path,), + ) + conn.commit() + conn.close() + if with_canonical: + db.set_album_canonical("alb1", "spotify", "sp_evolve", 0.96) + + +def test_returns_canonical_when_pinned(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + fp = "/music/Evolve/01 - Believer.flac" + _seed(db, with_canonical=True, file_path=fp) + assert _lookup_canonical_from_db([(fp, "01 - Believer.flac", 1)], _ctx(db)) == ("spotify", "sp_evolve") + + +def test_none_when_unresolved(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + fp = "/music/Evolve/01 - Believer.flac" + _seed(db, with_canonical=False, file_path=fp) + assert _lookup_canonical_from_db([(fp, "01 - Believer.flac", 1)], _ctx(db)) is None + + +def test_none_when_file_not_tracked(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + _seed(db, with_canonical=True) + assert _lookup_canonical_from_db([("/some/other/path.flac", "x.flac", 1)], _ctx(db)) is None + + +def test_none_when_no_db(): + assert _lookup_canonical_from_db([("/p.flac", "p.flac", 1)], types.SimpleNamespace(db=None)) is None From f9271c0cd8597c65bf636e7ff5656d2ad46de060 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 11:53:45 -0700 Subject: [PATCH 014/108] =?UTF-8?q?Canonical=20album=20version=20=E2=80=94?= =?UTF-8?q?=20backfill=20job=20(the=20opt-in=20activation)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The populate trigger that turns the (until now dormant) feature on. Until a user enables and runs this job, no album has a canonical -> both read sides (Stages 3-4) fall back -> zero behavior change. So the whole feature ships safely off. - core/repair_jobs/canonical_version_resolve.py — "Resolve Canonical Album Versions". Iterates the active server's albums, skips ones already pinned, and calls the tested resolve_and_store_canonical_for_album per album. Opt-in (default_enabled=False) and dry-run-by-default: resolving compares an album's candidate releases across sources (metadata-source API calls, once per album), so it's deliberately user-triggered. Dry run reports a finding per album it would pin; live mode stores. Registered in _JOB_MODULES. - core/metadata/canonical_resolver.py — resolve_and_store gains store=True; the job's dry run passes store=False to resolve-without-writing. Tests: tests/test_canonical_version_job.py (6) — registered, opt-in + dry-run defaults, live resolves+stores (auto_fixed), dry run creates findings without persisting, already-pinned albums skipped. Registry loads all 19 jobs cleanly. 145 tests across the full feature + reorganize/track-repair/DB regression pass. --- core/metadata/canonical_resolver.py | 8 +- core/repair_jobs/__init__.py | 1 + core/repair_jobs/canonical_version_resolve.py | 164 ++++++++++++++++++ tests/test_canonical_version_job.py | 103 +++++++++++ 4 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 core/repair_jobs/canonical_version_resolve.py create mode 100644 tests/test_canonical_version_job.py diff --git a/core/metadata/canonical_resolver.py b/core/metadata/canonical_resolver.py index a240870a..15a1b279 100644 --- a/core/metadata/canonical_resolver.py +++ b/core/metadata/canonical_resolver.py @@ -110,10 +110,12 @@ def resolve_and_store_canonical_for_album( fetch_tracklist: Optional[Callable[[str, str], Any]] = None, source_priority: Optional[List[str]] = None, min_score: float = 0.5, + store: bool = True, ) -> Optional[Dict[str, Any]]: """Gather an album's source IDs + its tracks' (duration, title) from the DB, - resolve the best-fit canonical release, and persist it. Returns the stored - ``{source, album_id, score}`` or None when unresolved. + resolve the best-fit canonical release, and (when ``store``) persist it. + Returns the resolved ``{source, album_id, score}`` or None when unresolved. + ``store=False`` resolves without writing — used by the backfill job's dry run. Uses the SAME album/source-id loader the Reorganizer uses (``load_album_and_tracks`` + ``_extract_source_ids``) so the canonical is @@ -150,7 +152,7 @@ def resolve_and_store_canonical_for_album( source_priority=source_priority, min_score=min_score, ) - if result: + if result and store: db.set_album_canonical(album_id, result['source'], result['album_id'], result['score']) return result diff --git a/core/repair_jobs/__init__.py b/core/repair_jobs/__init__.py index c6f780e8..40fd7a04 100644 --- a/core/repair_jobs/__init__.py +++ b/core/repair_jobs/__init__.py @@ -44,6 +44,7 @@ _JOB_MODULES = [ 'core.repair_jobs.live_commentary_cleaner', 'core.repair_jobs.unknown_artist_fixer', 'core.repair_jobs.discography_backfill', + 'core.repair_jobs.canonical_version_resolve', ] diff --git a/core/repair_jobs/canonical_version_resolve.py b/core/repair_jobs/canonical_version_resolve.py new file mode 100644 index 00000000..1323eb11 --- /dev/null +++ b/core/repair_jobs/canonical_version_resolve.py @@ -0,0 +1,164 @@ +"""Resolve Canonical Album Versions — backfill job (#765 Stage 2 trigger). + +Pins each album's canonical release (best-fit to its files) so the Library +Reorganizer (Stage 3) and Track Number Repair (Stage 4) resolve the SAME +release and stop contradicting each other. The resolution logic lives in the +tested core.metadata.canonical_resolver; this job is the opt-in, rate-limited, +progress-reported bulk runner. + +Opt-in (``default_enabled = False``) because resolving compares an album's +candidate releases across sources, which costs metadata-source API calls — done +once per album, then stored. Albums that already have a canonical are skipped. +""" + +import os +from typing import Optional + +from core.metadata.canonical_resolver import resolve_and_store_canonical_for_album +from core.repair_jobs import register_job +from core.repair_jobs.base import JobContext, JobResult, RepairJob +from utils.logging_config import get_logger + +logger = get_logger("repair_job.canonical_version") + + +@register_job +class CanonicalVersionResolveJob(RepairJob): + job_id = 'canonical_version_resolve' + display_name = 'Resolve Canonical Album Versions' + description = ( + 'Pins the best-fit release per album (by track count + durations) so ' + 'reorganize and track-number repair agree (dry run by default)' + ) + help_text = ( + 'For each album, compares the releases its linked metadata sources point ' + 'at and pins the one that best matches the files you actually have ' + '(track count + durations + titles). The Library Reorganizer and Track ' + 'Number Repair then both use that pinned release, so they stop ' + 'contradicting each other (e.g. standard vs deluxe, or Spotify vs ' + 'MusicBrainz track numbering).\n\n' + 'In dry run mode (default) it reports what it would pin without saving. ' + 'Disable dry run to store the pins. Albums already pinned are skipped.\n\n' + 'Opt-in: resolving costs metadata-source API calls (once per album).' + ) + icon = 'repair-icon-tracknumber' + default_enabled = False + default_interval_hours = 168 # weekly, but disabled by default + default_settings = { + 'dry_run': True, + 'min_score': 0.5, + } + auto_fix = True + + def _get_settings(self, context: JobContext) -> dict: + merged = dict(self.default_settings) + if context.config_manager: + merged.update(context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {}) or {}) + return merged + + def _load_album_ids(self, db, active_server: Optional[str]) -> list: + conn = None + try: + conn = db._get_connection() + cursor = conn.cursor() + if active_server: + cursor.execute( + "SELECT al.id, al.title FROM albums al WHERE al.server_source = ? ORDER BY al.id", + (active_server,), + ) + else: + cursor.execute("SELECT al.id, al.title FROM albums al ORDER BY al.id") + return [(row[0], row[1]) for row in cursor.fetchall()] + except Exception as e: + logger.error("Error loading albums for canonical resolve: %s", e) + return [] + finally: + if conn: + conn.close() + + def scan(self, context: JobContext) -> JobResult: + result = JobResult() + settings = self._get_settings(context) + dry_run = settings.get('dry_run', True) + min_score = settings.get('min_score', 0.5) + + active_server = None + if context.config_manager: + try: + active_server = context.config_manager.get_active_media_server() + except Exception as e: + logger.warning("Couldn't read active media server: %s", e) + + albums = self._load_album_ids(context.db, active_server) + total = len(albums) + if context.report_progress: + mode = 'DRY RUN' if dry_run else 'LIVE' + context.report_progress( + phase=f'Resolving canonical versions for {total} albums ({mode})...', + total=total, scanned=0, log_type='info', + ) + + for i, (album_id, album_title) in enumerate(albums): + if context.check_stop(): + return result + if i % 20 == 0 and context.wait_if_paused(): + return result + + # Skip albums already pinned — one-time cost per album. + try: + if context.db.get_album_canonical(album_id): + result.skipped += 1 + result.scanned += 1 + continue + except Exception: + pass + + try: + resolved = resolve_and_store_canonical_for_album( + context.db, album_id, min_score=min_score, store=not dry_run, + ) + except Exception as e: + logger.warning("Canonical resolve failed for album %s ('%s'): %s", + album_id, album_title, e) + result.errors += 1 + result.scanned += 1 + continue + + result.scanned += 1 + if resolved: + if dry_run and context.create_finding: + inserted = context.create_finding( + job_id=self.job_id, + finding_type='canonical_version', + severity='info', + entity_type='album', + entity_id=str(album_id), + file_path=None, + title=f'Would pin canonical: {album_title or album_id}', + description=( + f"Best-fit release: {resolved['source']} " + f"({resolved['album_id']}), score {resolved['score']}" + ), + details={'album_id': str(album_id), **resolved}, + ) + if inserted: + result.findings_created += 1 + else: + result.findings_skipped_dedup += 1 + elif not dry_run: + result.auto_fixed += 1 + + if context.report_progress and (i + 1) % 25 == 0: + context.report_progress(scanned=i + 1, total=total, + phase=f'Resolving ({i+1}/{total})...') + + return result + + def estimate_scope(self, context: JobContext) -> int: + active_server = None + if context.config_manager: + try: + active_server = context.config_manager.get_active_media_server() + except Exception: + pass + return len(self._load_album_ids(context.db, active_server)) diff --git a/tests/test_canonical_version_job.py b/tests/test_canonical_version_job.py new file mode 100644 index 00000000..32c20e3f --- /dev/null +++ b/tests/test_canonical_version_job.py @@ -0,0 +1,103 @@ +"""Backfill job: Resolve Canonical Album Versions (#765 Stage 2 trigger).""" + +from __future__ import annotations + +import types + +import core.repair_jobs.canonical_version_resolve as cvr +from core.repair_jobs import get_all_jobs +from core.repair_jobs.canonical_version_resolve import CanonicalVersionResolveJob +from database.music_database import MusicDatabase + + +def _ctx(db, findings): + return types.SimpleNamespace( + db=db, + config_manager=None, # -> active_server None -> all albums + check_stop=lambda: False, + wait_if_paused=lambda: False, + report_progress=None, + update_progress=None, + create_finding=lambda **kw: (findings.append(kw) or True), + ) + + +def _seed_two_albums(db): + conn = db._get_connection() + cur = conn.cursor() + cur.execute("INSERT INTO artists (id, name) VALUES ('art1', 'A')") + cur.execute("INSERT INTO albums (id, title, artist_id) VALUES ('alb1', 'Album One', 'art1')") + cur.execute("INSERT INTO albums (id, title, artist_id) VALUES ('alb2', 'Album Two', 'art1')") + conn.commit() + conn.close() + + +def _fake_resolver(monkeypatch): + def fake(db, album_id, *, min_score=0.5, store=True): + res = {"source": "spotify", "album_id": f"sp_{album_id}", "score": 0.9} + if store: + db.set_album_canonical(album_id, res["source"], res["album_id"], res["score"]) + return res + monkeypatch.setattr(cvr, "resolve_and_store_canonical_for_album", fake) + + +def test_job_is_registered(): + jobs = get_all_jobs() # {job_id: cls} + assert "canonical_version_resolve" in jobs + assert jobs["canonical_version_resolve"] is CanonicalVersionResolveJob + + +def test_job_is_opt_in_and_dry_run_by_default(): + assert CanonicalVersionResolveJob.default_enabled is False + assert CanonicalVersionResolveJob.default_settings["dry_run"] is True + + +def test_live_resolves_and_stores(tmp_path, monkeypatch): + db = MusicDatabase(str(tmp_path / "m.db")) + _seed_two_albums(db) + _fake_resolver(monkeypatch) + + findings = [] + ctx = _ctx(db, findings) + job = CanonicalVersionResolveJob() + # force live mode + monkeypatch.setattr(job, "_get_settings", lambda c: {"dry_run": False, "min_score": 0.5}) + + result = job.scan(ctx) + assert result.auto_fixed == 2 + assert db.get_album_canonical("alb1")["source"] == "spotify" + assert db.get_album_canonical("alb2")["album_id"] == "sp_alb2" + assert findings == [] # live mode writes, doesn't create findings + + +def test_dry_run_creates_findings_without_storing(tmp_path, monkeypatch): + db = MusicDatabase(str(tmp_path / "m.db")) + _seed_two_albums(db) + _fake_resolver(monkeypatch) + + findings = [] + ctx = _ctx(db, findings) + job = CanonicalVersionResolveJob() + monkeypatch.setattr(job, "_get_settings", lambda c: {"dry_run": True, "min_score": 0.5}) + + result = job.scan(ctx) + assert result.findings_created == 2 + assert len(findings) == 2 + # dry run must NOT persist + assert db.get_album_canonical("alb1") is None + + +def test_skips_already_pinned_albums(tmp_path, monkeypatch): + db = MusicDatabase(str(tmp_path / "m.db")) + _seed_two_albums(db) + db.set_album_canonical("alb1", "deezer", "dz_pinned", 0.8) # alb1 already pinned + _fake_resolver(monkeypatch) + + ctx = _ctx(db, []) + job = CanonicalVersionResolveJob() + monkeypatch.setattr(job, "_get_settings", lambda c: {"dry_run": False, "min_score": 0.5}) + + result = job.scan(ctx) + assert result.skipped == 1 # alb1 skipped + assert result.auto_fixed == 1 # only alb2 resolved + assert db.get_album_canonical("alb1")["album_id"] == "dz_pinned" # untouched From e40b328a941645bfc0def1bbae170e3c16fa7982 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 12:41:36 -0700 Subject: [PATCH 015/108] docs: canonical-album-version design spec The staged design doc for this branch (#765 + #767-Bug2): the match-your-files canonical rule, the additive/dormant rollout, and the stage-by-stage plan the 6 implementation commits followed. Kept on the branch as its reference; not relevant to dev/main. --- SPEC_canonical_album_version.md | 144 ++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 SPEC_canonical_album_version.md diff --git a/SPEC_canonical_album_version.md b/SPEC_canonical_album_version.md new file mode 100644 index 00000000..fafb05bd --- /dev/null +++ b/SPEC_canonical_album_version.md @@ -0,0 +1,144 @@ +# Spec: Canonical Album Version (fixes #765 + #767-Bug2) + +**Status:** design only — no code yet. +**Goal:** Pin ONE canonical `(source, album_id)` per album, chosen by best-fit to +the user's actual files, so the Library Reorganizer, Track Number Repair, and +tagging/enrichment all agree on the same release. Today each re-resolves +independently and they contradict each other (Spotify Believer=4 vs MusicBrainz +Believer=3; standard album mislinked to a deluxe release). + +**Canonical-selection rule (decided):** *match the user's actual files.* The +canonical release is the candidate whose track count + per-track durations + +titles best fit what's on disk. Self-correcting: picks standard when you own the +standard, deluxe when you own the deluxe. + +--- + +## Hard requirement: don't disrupt the running app + +Every stage below is **additive and dormant until explicitly consumed**, and +every consumer **falls back to today's behavior when no canonical is set**. So: +- albums with no resolved canonical behave EXACTLY as they do now; +- each stage is independently shippable and reversible; +- nothing big-bangs. + +--- + +## Stage 1 — Schema + pure scorer (ships dormant, zero behavior change) + +### Schema (additive, nullable → migration-safe) +Add to `albums` (guarded `ALTER TABLE ... ADD COLUMN`, idempotent — mirror the +existing column-exists checks; see [[db-schema-review]] migration-safety notes): +- `canonical_source TEXT` — e.g. 'spotify' / 'itunes' / 'musicbrainz' +- `canonical_album_id TEXT` +- `canonical_score REAL` — best-fit score (for transparency / re-resolve gating) +- `canonical_resolved_at TIMESTAMP` + +All nullable. Existing rows = NULL → "unresolved" → consumers fall back. No +backfill in this stage. No reads in this stage. + +### Pure core helper (the testable heart) — `core/metadata/canonical_version.py` +``` +score_release_against_files(file_tracks, release_tracks) -> float +pick_canonical_release(file_tracks, candidates) -> (best, score) | (None, 0) +``` +- `file_tracks`: list of {duration_ms, title, track_number?} read from disk. +- `release_tracks`: a candidate release's tracklist (same shape). +- Scoring (tunable weights): + - **track-count fit** — exact match strongly preferred; |Δcount| penalized. + - **duration alignment** — greedily match each file to its closest release + track by duration (within a tolerance, e.g. ±3s); reward coverage. + - **title overlap** — token/fuzzy overlap as a tiebreaker. + - **graceful degradation** — if a source gives no per-track durations, fall + back to count + title only (never crash, never force-pick). +- Returns the best candidate + score, or (None, 0) when nothing clears a floor + (so we never pin a bad guess — leave it unresolved, consumers fall back). + +### Tests (extreme, like the rest of this codebase) +- standard (11) vs deluxe (17) with 11 files on disk → picks standard. +- same album, 17 files → picks deluxe. +- duration disambiguation when track counts tie (e.g. radio edit vs album). +- missing-duration source → count+title fallback still picks sanely. +- no candidate clears the floor → (None, 0). +- "Believer" standard(=track 3 listing) vs Spotify(=4) with the user's files → + whichever the files actually match. + +**End of Stage 1: scorer exists + tested, columns exist, NOTHING reads/writes +them yet. Provably zero behavior change.** + +--- + +## Stage 2 — Resolver populates canonical (writes, still no consumers) + +A function `resolve_canonical_for_album(album_id, db, ...)`: +1. Gather on-disk file metadata for the album (durations/titles) via the + library's known file paths. +2. Gather candidate releases: every source the album has an ID for + (spotify/itunes/deezer/discogs/soul/musicbrainz) AND — for the deluxe/standard + case — sibling editions discoverable from those. Fetch each tracklist + (cached, rate-limited). +3. `pick_canonical_release(files, candidates)` → store `(source, album_id, score)` + on the album row if it clears the floor. + +Wiring: a small **backfill repair job** (dry-run-capable) + a hook in enrichment +when an album is (re)enriched. Still **no tool READS canonical**, so behavior is +unchanged — this stage only populates the new columns. Reversible: clearing the +columns reverts to unresolved. + +Tests: resolver picks the right release for the standard/deluxe fixtures; stores +nothing when below floor; idempotent re-resolve. + +Cost note: fetching multiple candidate releases = more API calls. Mitigate via +cache + only-on-(re)enrich + the existing rate trackers. Surface in the job's +progress so it's not silent. + +--- + +## Stage 3 — Reorganizer reads canonical (first real behavior change, gated) + +In `library_reorganize._resolve_source`: if the album has +`canonical_source`/`canonical_album_id`, use THAT first; else fall back to the +current `get_source_priority` walk. One-line precedence change, fully gated on +non-NULL. + +Tests: with canonical set → resolves to it; with canonical NULL → byte-identical +to today. Re-run the existing reorganize battery (148 tests) — must stay green. + +**This alone fixes #767-Bug2** (a standard album whose files match the standard +release pins the standard, so reorganize stops targeting the deluxe folder). + +--- + +## Stage 4 — Track Number Repair reads canonical (closes #765) + +In `track_number_repair._resolve_album_tracklist`: add **Fallback -1** (before +everything) — if the album has a canonical `(source, album_id)`, use it. The +existing 6-level cascade stays as the fallback for albums with no canonical +(preserves its all-01-album rescue ability — the regression risk we refused to +take in the reactive fix). + +Now both tools resolve the SAME release → same track numbers → no contradiction. + +Tests: canonical present → both tools agree (shared-release test); canonical +NULL → existing cascade unchanged. + +--- + +## Risks & mitigations +- **Extra API calls** (Stage 2 fetches multiple releases) → cache, rate-limit, + only-on-(re)enrich, progress-logged. +- **Sources without per-track durations** → scorer degrades to count+title. +- **Schema migration** → additive nullable columns only; idempotent guards. +- **Wrong pick** → floor gate (never pin a low-confidence guess); `canonical_score` + stored for inspection/re-resolve; manual override possible later. +- **Backward-compat** → every consumer falls back to today's path when NULL, so + un-resolved albums (incl. all existing albums until backfilled) are unaffected. + +## Out of scope (for now) +- Per-album manual version override UI (can layer on later — the columns support it). +- Merging the two tools into one (the reporter's alt suggestion) — unnecessary + once they share the canonical. + +## Suggested order to build +1, then 2, then 3, then 4 — each shippable and verifiable on its own. We can stop +after any stage and the app is consistent (just with fewer consumers wired). From 57e039e34d0ba577017364c74a59a2230c88a13c Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 12:58:59 -0700 Subject: [PATCH 016/108] Canonical: make source selection a job setting (default active-preferred) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feedback from the live dry-run: the job was pinning whichever source best fit the files regardless of which source it was, which was surprising — users expect it to respect their active metadata source. Made it a per-job setting instead of a baked-in policy. source_selection (default 'active_preferred'): - active_preferred — use the active/primary metadata source's release when the album has an ID for it AND it clears the score floor; otherwise fall back to the best-fit among the other sources. Respects the configured source but self-heals when that link is clearly broken (below floor / no ID). - active_only — only ever the active source; never considers others. - best_fit — previous behavior: whichever source matches the files best. resolve_canonical_for_album gains mode + primary_source; the orchestration threads the primary source through; the job reads source_selection from its settings. Note: active_preferred respects the active source as long as it clears the floor, so it will NOT override a deluxe-vs-standard mismatch on the primary (#767-Bug2) — that's what best_fit is for; the choice is now the user's. Tests: per-mode coverage in test_canonical_resolver.py (active_preferred uses primary when it fits, falls back when primary is below floor, keeps primary even when another fits better; active_only pins primary / never falls back; best_fit unchanged), orchestration default-mode test, and the setting default. 39 canonical tests pass. --- core/metadata/canonical_resolver.py | 78 +++++++++++++++---- core/repair_jobs/canonical_version_resolve.py | 8 +- tests/test_canonical_orchestration.py | 17 +++- tests/test_canonical_resolver.py | 77 +++++++++++++++++- tests/test_canonical_version_job.py | 6 +- 5 files changed, 162 insertions(+), 24 deletions(-) diff --git a/core/metadata/canonical_resolver.py b/core/metadata/canonical_resolver.py index 15a1b279..86ee6e51 100644 --- a/core/metadata/canonical_resolver.py +++ b/core/metadata/canonical_resolver.py @@ -17,7 +17,16 @@ from __future__ import annotations from typing import Any, Callable, Dict, List, Optional -from core.metadata.canonical_version import pick_canonical_release +from core.metadata.canonical_version import ( + pick_canonical_release, + score_release_against_files, +) + +# Source-selection modes (a per-job setting). See resolve_canonical_for_album. +MODE_ACTIVE_PREFERRED = "active_preferred" # default: use the active source if it fits, else best-fit +MODE_ACTIVE_ONLY = "active_only" # only ever the active source +MODE_BEST_FIT = "best_fit" # whichever source fits the files best +VALID_MODES = (MODE_ACTIVE_PREFERRED, MODE_ACTIVE_ONLY, MODE_BEST_FIT) def resolve_canonical_for_album( @@ -27,38 +36,68 @@ def resolve_canonical_for_album( fetch_tracklist: Callable[[str, str], Optional[List[Dict[str, Any]]]], source_priority: List[str], min_score: float = 0.5, + mode: str = MODE_ACTIVE_PREFERRED, + primary_source: Optional[str] = None, ) -> Optional[Dict[str, Any]]: - """Pick the canonical release for one album. + """Pick the canonical release for one album, honoring the source-selection mode. ``album_source_ids``: ``{source: album_id}`` the album is linked to. ``file_tracks``: on-disk track metadata (``{duration_ms, title}``). ``fetch_tracklist(source, album_id)``: returns that release's tracklist (or None/[] on miss); injected so callers supply ``get_album_tracks_for_source`` while tests supply a fake. - ``source_priority``: order to build candidates in — ties break toward the - earlier (higher-priority) source, keeping the choice deterministic. + ``source_priority``: source order; ties break toward the earlier source. + ``primary_source``: the user's active metadata source (defaults to the first + of ``source_priority``). - Returns ``{'source', 'album_id', 'score'}`` for the best fit, or ``None`` - when there are no files, no resolvable candidates, or nothing clears - ``min_score`` (caller leaves the album unresolved → tools fall back).""" + Modes: + - ``active_preferred`` (default): use the active source's release when the + album has an ID for it AND it clears ``min_score``; otherwise fall back + to the best-fit among the remaining sources. So it normally respects the + user's configured source but self-heals when that link is clearly wrong. + - ``active_only``: only ever the active source (pinned if it clears the + floor; never considers other sources). + - ``best_fit``: whichever source's release best matches the files. + + Returns ``{'source', 'album_id', 'score'}`` or ``None`` when there are no + files, no resolvable candidates, or nothing clears ``min_score``.""" if not file_tracks: return None + primary = primary_source or (source_priority[0] if source_priority else None) - candidates: List[Dict[str, Any]] = [] - for source in source_priority: + def _candidate(source: Optional[str]) -> Optional[Dict[str, Any]]: + if not source: + return None album_id = album_source_ids.get(source) if not album_id: - continue + return None try: tracks = fetch_tracklist(source, str(album_id)) except Exception: tracks = None - if tracks: - candidates.append({ - 'source': source, - 'album_id': str(album_id), - 'tracks': tracks, - }) + if not tracks: + return None + return {'source': source, 'album_id': str(album_id), 'tracks': tracks} + + # Active-source modes: try the primary first. + if mode in (MODE_ACTIVE_ONLY, MODE_ACTIVE_PREFERRED): + cand = _candidate(primary) + if cand: + score = score_release_against_files(file_tracks, cand['tracks']) + if score >= min_score: + return {'source': cand['source'], 'album_id': cand['album_id'], 'score': round(score, 4)} + if mode == MODE_ACTIVE_ONLY: + return None # never fall back to other sources + + # best_fit (and active_preferred's fallback): score the candidate sources + # (skipping the primary we already tried in active_preferred) and pick best. + candidates: List[Dict[str, Any]] = [] + for source in source_priority: + if mode == MODE_ACTIVE_PREFERRED and source == primary: + continue + cand = _candidate(source) + if cand: + candidates.append(cand) if not candidates: return None @@ -111,6 +150,7 @@ def resolve_and_store_canonical_for_album( source_priority: Optional[List[str]] = None, min_score: float = 0.5, store: bool = True, + mode: str = MODE_ACTIVE_PREFERRED, ) -> Optional[Dict[str, Any]]: """Gather an album's source IDs + its tracks' (duration, title) from the DB, resolve the best-fit canonical release, and (when ``store``) persist it. @@ -138,10 +178,12 @@ def resolve_and_store_canonical_for_album( if fetch_tracklist is None: fetch_tracklist = default_fetch_tracklist + primary_source = None if source_priority is None: try: from core.metadata_service import get_primary_source, get_source_priority - source_priority = get_source_priority(get_primary_source()) + primary_source = get_primary_source() + source_priority = get_source_priority(primary_source) except Exception: source_priority = list(source_ids.keys()) @@ -151,6 +193,8 @@ def resolve_and_store_canonical_for_album( fetch_tracklist=fetch_tracklist, source_priority=source_priority, min_score=min_score, + mode=mode, + primary_source=primary_source, ) if result and store: db.set_album_canonical(album_id, result['source'], result['album_id'], result['score']) diff --git a/core/repair_jobs/canonical_version_resolve.py b/core/repair_jobs/canonical_version_resolve.py index 1323eb11..79c4aaef 100644 --- a/core/repair_jobs/canonical_version_resolve.py +++ b/core/repair_jobs/canonical_version_resolve.py @@ -47,6 +47,11 @@ class CanonicalVersionResolveJob(RepairJob): default_settings = { 'dry_run': True, 'min_score': 0.5, + # Which source's release to pin: 'active_preferred' (default — use your + # active metadata source when it fits, else best-fit fallback), + # 'active_only' (only ever the active source), or 'best_fit' (whichever + # source matches the files best, regardless of which it is). + 'source_selection': 'active_preferred', } auto_fix = True @@ -81,6 +86,7 @@ class CanonicalVersionResolveJob(RepairJob): settings = self._get_settings(context) dry_run = settings.get('dry_run', True) min_score = settings.get('min_score', 0.5) + mode = settings.get('source_selection', 'active_preferred') active_server = None if context.config_manager: @@ -115,7 +121,7 @@ class CanonicalVersionResolveJob(RepairJob): try: resolved = resolve_and_store_canonical_for_album( - context.db, album_id, min_score=min_score, store=not dry_run, + context.db, album_id, min_score=min_score, store=not dry_run, mode=mode, ) except Exception as e: logger.warning("Canonical resolve failed for album %s ('%s'): %s", diff --git a/tests/test_canonical_orchestration.py b/tests/test_canonical_orchestration.py index 2f3ae8a0..44ae1b76 100644 --- a/tests/test_canonical_orchestration.py +++ b/tests/test_canonical_orchestration.py @@ -47,14 +47,29 @@ def test_resolve_and_store_picks_best_fit_and_persists(tmp_path): db, album_id, fetch_tracklist=lambda s, a: table.get((s, a)), source_priority=["spotify", "deezer"], + mode="best_fit", ) - # Deezer's standard matches the 11 files better than Spotify's deluxe. + # best_fit: Deezer's standard matches the 11 files better than Spotify's deluxe. assert out["source"] == "deezer" and out["album_id"] == "dz_std" # ...and it was persisted. stored = db.get_album_canonical(album_id) assert stored["source"] == "deezer" and stored["album_id"] == "dz_std" +def test_default_mode_prefers_active_source(tmp_path): + # Same setup, but default (active_preferred) mode: primary = spotify, whose + # deluxe still clears the floor -> pinned, even though deezer fits better. + db = MusicDatabase(str(tmp_path / "m.db")) + album_id = _seed(db, spotify="sp_deluxe", deezer="dz_std") + table = {("spotify", "sp_deluxe"): DLX, ("deezer", "dz_std"): STD} + out = resolve_and_store_canonical_for_album( + db, album_id, + fetch_tracklist=lambda s, a: table.get((s, a)), + source_priority=["spotify", "deezer"], # default mode + ) + assert out["source"] == "spotify" # active source preferred + + def test_resolve_returns_none_when_album_has_no_source_ids(tmp_path): db = MusicDatabase(str(tmp_path / "m.db")) album_id = _seed(db, spotify=None, deezer=None) diff --git a/tests/test_canonical_resolver.py b/tests/test_canonical_resolver.py index 2868a373..a6e7839e 100644 --- a/tests/test_canonical_resolver.py +++ b/tests/test_canonical_resolver.py @@ -17,10 +17,10 @@ def _fetcher(table): return fetch -def test_picks_source_whose_release_fits_the_files(): +def test_best_fit_mode_picks_best_regardless_of_priority(): files = list(STD) # user owns the 11-track standard table = { - ("spotify", "sp_deluxe"): DLX, # spotify linked to deluxe (17) + ("spotify", "sp_deluxe"): DLX, # spotify (primary) linked to deluxe (17) ("musicbrainz", "mb_std"): STD, # musicbrainz has standard (11) } out = resolve_canonical_for_album( @@ -28,12 +28,81 @@ def test_picks_source_whose_release_fits_the_files(): file_tracks=files, fetch_tracklist=_fetcher(table), source_priority=PRIORITY, + mode="best_fit", ) - # Best FIT wins over priority — standard matches the files, deluxe doesn't. - assert out == {"source": "musicbrainz", "album_id": "mb_std", "score": out["score"]} + # best_fit: standard matches the files, deluxe doesn't — fit beats priority. + assert out["source"] == "musicbrainz" and out["album_id"] == "mb_std" assert out["score"] > 0.9 +# ── source-selection modes ──────────────────────────────────────────────── + +def test_active_preferred_uses_primary_when_it_fits(): + files = list(STD) + table = {("spotify", "sp1"): STD, ("musicbrainz", "mb1"): STD} # both fit + out = resolve_canonical_for_album( + album_source_ids={"spotify": "sp1", "musicbrainz": "mb1"}, + file_tracks=files, fetch_tracklist=_fetcher(table), + source_priority=PRIORITY, # primary = spotify + ) # default mode = active_preferred + assert out["source"] == "spotify" + + +def test_active_preferred_falls_back_when_primary_clearly_misfits(): + files = list(STD) # 11 tracks + table = { + ("spotify", "sp_bad"): [{"duration_ms": 60_000, "title": "X"}] * 3, # 3-track, fall back to the fitting source. + assert out["source"] == "musicbrainz" + + +def test_active_preferred_keeps_primary_even_if_another_fits_better(): + files = list(STD) + # primary spotify is a deluxe (decent fit, above floor); musicbrainz is exact. + table = {("spotify", "sp_dlx"): DLX, ("musicbrainz", "mb_std"): STD} + out = resolve_canonical_for_album( + album_source_ids={"spotify": "sp_dlx", "musicbrainz": "mb_std"}, + file_tracks=files, fetch_tracklist=_fetcher(table), + source_priority=PRIORITY, mode="active_preferred", + ) + # active_preferred respects the active source as long as it clears the floor, + # even though musicbrainz would fit better (use best_fit for that). + assert out["source"] == "spotify" + + +def test_active_only_pins_primary_and_never_falls_back(): + files = list(STD) + # primary spotify is below floor; a perfect musicbrainz exists but is ignored. + table = { + ("spotify", "sp_bad"): [{"duration_ms": 60_000, "title": "X"}] * 3, + ("musicbrainz", "mb_std"): STD, + } + out = resolve_canonical_for_album( + album_source_ids={"spotify": "sp_bad", "musicbrainz": "mb_std"}, + file_tracks=files, fetch_tracklist=_fetcher(table), + source_priority=PRIORITY, mode="active_only", + ) + assert out is None # primary didn't fit, and active_only won't consider others + + +def test_active_only_pins_primary_when_it_fits(): + files = list(STD) + table = {("spotify", "sp1"): STD, ("musicbrainz", "mb1"): STD} + out = resolve_canonical_for_album( + album_source_ids={"spotify": "sp1", "musicbrainz": "mb1"}, + file_tracks=files, fetch_tracklist=_fetcher(table), + source_priority=PRIORITY, mode="active_only", + ) + assert out["source"] == "spotify" + + def test_priority_breaks_tie_between_equal_fits(): files = list(STD) table = {("spotify", "a"): STD, ("itunes", "b"): STD} # identical fit diff --git a/tests/test_canonical_version_job.py b/tests/test_canonical_version_job.py index 32c20e3f..cf04cd29 100644 --- a/tests/test_canonical_version_job.py +++ b/tests/test_canonical_version_job.py @@ -33,7 +33,7 @@ def _seed_two_albums(db): def _fake_resolver(monkeypatch): - def fake(db, album_id, *, min_score=0.5, store=True): + def fake(db, album_id, *, min_score=0.5, store=True, mode="active_preferred"): res = {"source": "spotify", "album_id": f"sp_{album_id}", "score": 0.9} if store: db.set_album_canonical(album_id, res["source"], res["album_id"], res["score"]) @@ -52,6 +52,10 @@ def test_job_is_opt_in_and_dry_run_by_default(): assert CanonicalVersionResolveJob.default_settings["dry_run"] is True +def test_source_selection_defaults_to_active_preferred(): + assert CanonicalVersionResolveJob.default_settings["source_selection"] == "active_preferred" + + def test_live_resolves_and_stores(tmp_path, monkeypatch): db = MusicDatabase(str(tmp_path / "m.db")) _seed_two_albums(db) From ec8091caada3e6429cab1462418aa11a97d1207b Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 13:13:37 -0700 Subject: [PATCH 017/108] Canonical: richer, judge-able findings (the why behind a pin) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live-run feedback: "Best-fit release: deezer (665666731), score 1.0" is too thin to trust/accept. Each finding now explains WHY: - score_release_detail() exposes the per-signal breakdown (count/duration/title) instead of just the blended score. - resolve_canonical_for_album returns an enriched result: the breakdown, file_track_count vs release_track_count, and a `candidates` list of every source it scored (so a finding can show what the winner beat). - resolve_and_store adds album/artist/thumb context from the row it already loaded (no extra query). Storage still only reads source/album_id/score. - The job builds a real description via _describe_pin(), e.g.: "Pin deezer release 665666731 (confidence 100%). Fit to your library: 11 files vs 11 tracks on this release — track count 100%, durations 100%, titles 100%. Beat: spotify 65% (17 tk)." and a clearer title ("Pin deezer as canonical: — "). Tests: resolver enrichment (breakdown + candidate comparison fields), and _describe_pin (judge-able text incl. the beaten alternatives, and honest "n/a" for a missing signal). 42 canonical tests pass. Note: the description string carries the judge-able info regardless of UI; how the findings tab renders the extra details keys (thumb image, candidates table) is still UI-dependent and unverified. --- core/metadata/canonical_resolver.py | 91 ++++++++++++------- core/metadata/canonical_version.py | 28 ++++++ core/repair_jobs/canonical_version_resolve.py | 35 ++++++- tests/test_canonical_resolver.py | 18 ++++ tests/test_canonical_version_job.py | 32 ++++++- 5 files changed, 166 insertions(+), 38 deletions(-) diff --git a/core/metadata/canonical_resolver.py b/core/metadata/canonical_resolver.py index 86ee6e51..8762bbd1 100644 --- a/core/metadata/canonical_resolver.py +++ b/core/metadata/canonical_resolver.py @@ -18,8 +18,8 @@ from __future__ import annotations from typing import Any, Callable, Dict, List, Optional from core.metadata.canonical_version import ( - pick_canonical_release, score_release_against_files, + score_release_detail, ) # Source-selection modes (a per-job setting). See resolve_canonical_for_album. @@ -59,15 +59,20 @@ def resolve_canonical_for_album( floor; never considers other sources). - ``best_fit``: whichever source's release best matches the files. - Returns ``{'source', 'album_id', 'score'}`` or ``None`` when there are no - files, no resolvable candidates, or nothing clears ``min_score``.""" + Returns an enriched dict for the chosen release — ``source``, ``album_id``, + ``score``, the per-signal breakdown (``count_fit``/``duration_fit``/ + ``title_fit``), ``file_track_count`` vs ``release_track_count``, and a + ``candidates`` list of everything it scored (so a finding can show WHY the + pick won and what it beat). ``None`` when there are no files, no resolvable + candidates, or nothing clears ``min_score``.""" if not file_tracks: return None primary = primary_source or (source_priority[0] if source_priority else None) + scored: List[Dict[str, Any]] = [] # every source we actually scored - def _candidate(source: Optional[str]) -> Optional[Dict[str, Any]]: - if not source: - return None + def _score(source: Optional[str]) -> Optional[Dict[str, Any]]: + if not source or any(e['source'] == source for e in scored): + return next((e for e in scored if e['source'] == source), None) album_id = album_source_ids.get(source) if not album_id: return None @@ -77,38 +82,53 @@ def resolve_canonical_for_album( tracks = None if not tracks: return None - return {'source': source, 'album_id': str(album_id), 'tracks': tracks} + entry = { + 'source': source, 'album_id': str(album_id), + 'track_count': len(tracks), 'score': round(score_release_against_files(file_tracks, tracks), 4), + '_tracks': tracks, + } + scored.append(entry) + return entry + + winner: Optional[Dict[str, Any]] = None # Active-source modes: try the primary first. if mode in (MODE_ACTIVE_ONLY, MODE_ACTIVE_PREFERRED): - cand = _candidate(primary) - if cand: - score = score_release_against_files(file_tracks, cand['tracks']) - if score >= min_score: - return {'source': cand['source'], 'album_id': cand['album_id'], 'score': round(score, 4)} - if mode == MODE_ACTIVE_ONLY: - return None # never fall back to other sources + p = _score(primary) + if p and p['score'] >= min_score: + winner = p + elif mode == MODE_ACTIVE_ONLY: + return None # never consider other sources - # best_fit (and active_preferred's fallback): score the candidate sources - # (skipping the primary we already tried in active_preferred) and pick best. - candidates: List[Dict[str, Any]] = [] - for source in source_priority: - if mode == MODE_ACTIVE_PREFERRED and source == primary: - continue - cand = _candidate(source) - if cand: - candidates.append(cand) + # best_fit, or active_preferred fallback: score the rest and pick the best. + if winner is None: + for source in source_priority: + _score(source) + best = None + for e in scored: # source_priority order -> strictly-greater = priority tiebreak + if best is None or e['score'] > best['score'] + 1e-9: + best = e + if best and best['score'] >= min_score: + winner = best - if not candidates: + if winner is None: return None - best, score = pick_canonical_release(file_tracks, candidates, min_score=min_score) - if not best: - return None + detail = score_release_detail(file_tracks, winner['_tracks']) return { - 'source': best['source'], - 'album_id': best['album_id'], - 'score': round(score, 4), + 'source': winner['source'], + 'album_id': winner['album_id'], + 'score': winner['score'], + 'file_track_count': detail['file_track_count'], + 'release_track_count': detail['release_track_count'], + 'count_fit': detail['count_fit'], + 'duration_fit': detail['duration_fit'], + 'title_fit': detail['title_fit'], + 'candidates': [ + {'source': e['source'], 'album_id': e['album_id'], + 'track_count': e['track_count'], 'score': e['score']} + for e in scored + ], } @@ -196,8 +216,15 @@ def resolve_and_store_canonical_for_album( mode=mode, primary_source=primary_source, ) - if result and store: - db.set_album_canonical(album_id, result['source'], result['album_id'], result['score']) + if result: + # Album/artist/art context for richer findings (read from the row we + # already loaded — no extra query). Storage only uses source/id/score. + result['album_title'] = album_data.get('title') or '' + result['artist_name'] = album_data.get('artist_name') or '' + if album_data.get('thumb_url'): + result['album_thumb_url'] = album_data['thumb_url'] + if store: + db.set_album_canonical(album_id, result['source'], result['album_id'], result['score']) return result diff --git a/core/metadata/canonical_version.py b/core/metadata/canonical_version.py index 0e44944e..ccc07451 100644 --- a/core/metadata/canonical_version.py +++ b/core/metadata/canonical_version.py @@ -145,6 +145,34 @@ def score_release_against_files( return _combine([(count, _W_COUNT), (dur, _W_DURATION), (title, _W_TITLE)]) +def score_release_detail( + file_tracks: List[Dict[str, Any]], + release_tracks: List[Dict[str, Any]], + *, + duration_tolerance_ms: int = _DEFAULT_DURATION_TOLERANCE_MS, +) -> Dict[str, Any]: + """Like ``score_release_against_files`` but returns the per-signal breakdown + so a UI can show WHY a release scored the way it did. ``duration_fit`` / + ``title_fit`` are ``None`` when that signal was absent.""" + if not file_tracks or not release_tracks: + return { + 'score': 0.0, 'count_fit': 0.0, 'duration_fit': None, 'title_fit': None, + 'release_track_count': len(release_tracks), 'file_track_count': len(file_tracks), + } + count = _count_fit(len(file_tracks), len(release_tracks)) + dur = _duration_fit(file_tracks, release_tracks, duration_tolerance_ms) + title = _title_fit(file_tracks, release_tracks) + score = _combine([(count, _W_COUNT), (dur, _W_DURATION), (title, _W_TITLE)]) + return { + 'score': round(score, 4), + 'count_fit': round(count, 3), + 'duration_fit': round(dur, 3) if dur is not None else None, + 'title_fit': round(title, 3) if title is not None else None, + 'release_track_count': len(release_tracks), + 'file_track_count': len(file_tracks), + } + + def pick_canonical_release( file_tracks: List[Dict[str, Any]], candidates: List[Dict[str, Any]], diff --git a/core/repair_jobs/canonical_version_resolve.py b/core/repair_jobs/canonical_version_resolve.py index 79c4aaef..6c802040 100644 --- a/core/repair_jobs/canonical_version_resolve.py +++ b/core/repair_jobs/canonical_version_resolve.py @@ -22,6 +22,32 @@ from utils.logging_config import get_logger logger = get_logger("repair_job.canonical_version") +def _pct(v) -> str: + return f"{round(v * 100)}%" if isinstance(v, (int, float)) else "n/a" + + +def _describe_pin(resolved: dict) -> str: + """Human-readable, judge-able explanation of WHY this release was chosen.""" + lines = [ + f"Pin {resolved['source']} release {resolved['album_id']} " + f"(confidence {_pct(resolved.get('score'))}).", + f"Fit to your library: {resolved.get('file_track_count', '?')} files vs " + f"{resolved.get('release_track_count', '?')} tracks on this release — " + f"track count {_pct(resolved.get('count_fit'))}, " + f"durations {_pct(resolved.get('duration_fit'))}, " + f"titles {_pct(resolved.get('title_fit'))}.", + ] + others = [c for c in resolved.get('candidates', []) if c.get('source') != resolved.get('source')] + if others: + comp = ", ".join( + f"{c['source']} {_pct(c['score'])} ({c['track_count']} tk)" for c in others + ) + lines.append(f"Beat: {comp}.") + elif len(resolved.get('candidates', [])) == 1: + lines.append("Only this source had a release linked for this album.") + return "\n".join(lines) + + @register_job class CanonicalVersionResolveJob(RepairJob): job_id = 'canonical_version_resolve' @@ -133,6 +159,8 @@ class CanonicalVersionResolveJob(RepairJob): result.scanned += 1 if resolved: if dry_run and context.create_finding: + artist = resolved.get('artist_name') or '' + label = f"{artist} — {album_title}" if artist else (album_title or str(album_id)) inserted = context.create_finding( job_id=self.job_id, finding_type='canonical_version', @@ -140,11 +168,8 @@ class CanonicalVersionResolveJob(RepairJob): entity_type='album', entity_id=str(album_id), file_path=None, - title=f'Would pin canonical: {album_title or album_id}', - description=( - f"Best-fit release: {resolved['source']} " - f"({resolved['album_id']}), score {resolved['score']}" - ), + title=f'Pin {resolved["source"]} as canonical: {label}', + description=_describe_pin(resolved), details={'album_id': str(album_id), **resolved}, ) if inserted: diff --git a/tests/test_canonical_resolver.py b/tests/test_canonical_resolver.py index a6e7839e..516eac11 100644 --- a/tests/test_canonical_resolver.py +++ b/tests/test_canonical_resolver.py @@ -92,6 +92,24 @@ def test_active_only_pins_primary_and_never_falls_back(): assert out is None # primary didn't fit, and active_only won't consider others +def test_result_includes_breakdown_and_candidate_comparison(): + files = list(STD) + table = {("spotify", "sp1"): DLX, ("deezer", "dz1"): STD} + out = resolve_canonical_for_album( + album_source_ids={"spotify": "sp1", "deezer": "dz1"}, + file_tracks=files, fetch_tracklist=_fetcher(table), + source_priority=["spotify", "deezer", "itunes", "musicbrainz"], + mode="best_fit", + ) + assert out["source"] == "deezer" + assert out["file_track_count"] == 11 + assert out["release_track_count"] == 11 + assert out["count_fit"] == 1.0 and out["duration_fit"] == 1.0 and out["title_fit"] == 1.0 + by_src = {c["source"]: c for c in out["candidates"]} + assert by_src["deezer"]["track_count"] == 11 and by_src["deezer"]["score"] > 0.9 + assert by_src["spotify"]["track_count"] == 17 and by_src["spotify"]["score"] < 0.8 + + def test_active_only_pins_primary_when_it_fits(): files = list(STD) table = {("spotify", "sp1"): STD, ("musicbrainz", "mb1"): STD} diff --git a/tests/test_canonical_version_job.py b/tests/test_canonical_version_job.py index cf04cd29..9e720adc 100644 --- a/tests/test_canonical_version_job.py +++ b/tests/test_canonical_version_job.py @@ -6,7 +6,10 @@ import types import core.repair_jobs.canonical_version_resolve as cvr from core.repair_jobs import get_all_jobs -from core.repair_jobs.canonical_version_resolve import CanonicalVersionResolveJob +from core.repair_jobs.canonical_version_resolve import ( + CanonicalVersionResolveJob, + _describe_pin, +) from database.music_database import MusicDatabase @@ -56,6 +59,33 @@ def test_source_selection_defaults_to_active_preferred(): assert CanonicalVersionResolveJob.default_settings["source_selection"] == "active_preferred" +def test_describe_pin_is_judgeable(): + desc = _describe_pin({ + "source": "deezer", "album_id": "665666731", "score": 1.0, + "file_track_count": 11, "release_track_count": 11, + "count_fit": 1.0, "duration_fit": 1.0, "title_fit": 1.0, + "candidates": [ + {"source": "deezer", "album_id": "665666731", "track_count": 11, "score": 1.0}, + {"source": "spotify", "album_id": "sp", "track_count": 17, "score": 0.65}, + ], + }) + assert "deezer" in desc and "665666731" in desc + assert "11 files vs 11 tracks" in desc # your library vs the release + assert "durations 100%" in desc and "titles 100%" in desc # the WHY + assert "Beat:" in desc and "spotify 65% (17 tk)" in desc # what it beat + + +def test_describe_pin_single_source(): + desc = _describe_pin({ + "source": "spotify", "album_id": "x", "score": 0.9, + "file_track_count": 10, "release_track_count": 10, + "count_fit": 1.0, "duration_fit": None, "title_fit": 0.9, + "candidates": [{"source": "spotify", "album_id": "x", "track_count": 10, "score": 0.9}], + }) + assert "Only this source" in desc + assert "durations n/a" in desc # missing signal shown honestly + + def test_live_resolves_and_stores(tmp_path, monkeypatch): db = MusicDatabase(str(tmp_path / "m.db")) _seed_two_albums(db) From 03d099fb1d426fe493bf1c33694cce6a55ca2db2 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 14:04:09 -0700 Subject: [PATCH 018/108] Canonical findings: add artist image (guarded, schema-safe) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Findings now carry artist_thumb_url alongside album_thumb_url (same key the track-repair findings use, so the findings UI already renders it). Fetched via a guarded _lookup_artist_thumb() — checks the artists table has a thumb_url column first and swallows any error — rather than adding ar.thumb_url to the shared load_album_and_tracks SELECT. The shared-loader approach was tried first and REVERTED: it crashed reorganize on schemas whose artists table has no thumb_url column (caught by 40 orchestrator tests). The lookup only runs for albums that actually resolve, so it adds no cost to the no-source-id short-circuit majority. Tests: orchestration test asserts artist_name + album_thumb_url + artist_thumb_url flow through. 47 canonical + 104 canonical/reorganize regression tests pass. --- core/metadata/canonical_resolver.py | 28 +++++++++++++++++++++++++++ tests/test_canonical_orchestration.py | 26 +++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/core/metadata/canonical_resolver.py b/core/metadata/canonical_resolver.py index 8762bbd1..b521dcb1 100644 --- a/core/metadata/canonical_resolver.py +++ b/core/metadata/canonical_resolver.py @@ -162,6 +162,28 @@ def default_fetch_tracklist(source: str, album_id: str) -> Optional[List[Dict[st return out or None +def _lookup_artist_thumb(db, artist_id) -> Optional[str]: + """Best-effort artist thumb URL by id. Returns None on missing column / any + error (the artists table doesn't have thumb_url in every schema).""" + if not artist_id: + return None + conn = None + try: + conn = db._get_connection() + cursor = conn.cursor() + cursor.execute("PRAGMA table_info(artists)") + if 'thumb_url' not in {r[1] for r in cursor.fetchall()}: + return None + cursor.execute("SELECT thumb_url FROM artists WHERE id = ?", (str(artist_id),)) + row = cursor.fetchone() + return (row[0] or None) if row else None + except Exception: + return None + finally: + if conn: + conn.close() + + def resolve_and_store_canonical_for_album( db, album_id, @@ -223,6 +245,12 @@ def resolve_and_store_canonical_for_album( result['artist_name'] = album_data.get('artist_name') or '' if album_data.get('thumb_url'): result['album_thumb_url'] = album_data['thumb_url'] + # Artist thumb via a guarded lookup (not the shared album loader — some + # schemas have no artists.thumb_url column). Only runs for resolved + # albums, so no cost on the no-source-id short-circuit majority. + artist_thumb = _lookup_artist_thumb(db, album_data.get('artist_id')) + if artist_thumb: + result['artist_thumb_url'] = artist_thumb if store: db.set_album_canonical(album_id, result['source'], result['album_id'], result['score']) return result diff --git a/tests/test_canonical_orchestration.py b/tests/test_canonical_orchestration.py index 44ae1b76..931302a3 100644 --- a/tests/test_canonical_orchestration.py +++ b/tests/test_canonical_orchestration.py @@ -70,6 +70,32 @@ def test_default_mode_prefers_active_source(tmp_path): assert out["source"] == "spotify" # active source preferred +def test_result_includes_artist_and_album_context(tmp_path): + db = MusicDatabase(str(tmp_path / "m.db")) + conn = db._get_connection() + cur = conn.cursor() + cur.execute("INSERT INTO artists (id, name, thumb_url) VALUES ('art1', 'Imagine Dragons', 'http://artist.jpg')") + cur.execute( + "INSERT INTO albums (id, title, artist_id, thumb_url, spotify_album_id) " + "VALUES ('alb1', 'Evolve', 'art1', 'http://album.jpg', 'sp1')" + ) + for i in range(11): + cur.execute( + "INSERT INTO tracks (id, album_id, artist_id, title, track_number, duration) " + "VALUES (?, 'alb1', 'art1', ?, ?, ?)", + (f"t{i}", f"Song {i+1}", i + 1, 180_000 + i * 10_000), + ) + conn.commit() + conn.close() + + out = resolve_and_store_canonical_for_album( + db, "alb1", fetch_tracklist=lambda s, a: STD, source_priority=["spotify"], + ) + assert out["artist_name"] == "Imagine Dragons" + assert out["album_thumb_url"] == "http://album.jpg" + assert out["artist_thumb_url"] == "http://artist.jpg" + + def test_resolve_returns_none_when_album_has_no_source_ids(tmp_path): db = MusicDatabase(str(tmp_path / "m.db")) album_id = _seed(db, spotify=None, deezer=None) From 2fcdfd3145d415355b9202c406d8985fda8eb003 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 14:10:02 -0700 Subject: [PATCH 019/108] Canonical findings: include as much (free) data as possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per request, pack each finding with everything available WITHOUT extra API calls (kettui: reuse what's already fetched, read the album row we already loaded, degrade per-field, keep it tested): - Pinned release's track titles — already fetched during scoring, so free (capped at 60 to bound details_json). - From the album row (free): year, DB track count, total duration, genres-free context, and the album's currently-linked source IDs. - file_track_titles (your library's titles) for a side-by-side with the release. - Artist + album thumbs (artist via the guarded lookup) and names. _describe_pin now renders: "Artist — Album (year)", the fit breakdown, "Currently linked: … → pinning X", "Beat: ", and the release tracklist — so the card is judge-able at a glance, and the structured fields are in details for a richer UI. NOT included (would cost an extra per-album API fetch, left as opt-in): the *release's* own year/type/cover/URL from get_album_for_source, vs the library's. Tests: _describe_pin rich-render (year/linked/tracklist), resolver release-titles, orchestration free-context fields. 94 canonical + reorganize regression pass. --- core/metadata/canonical_resolver.py | 14 +++++++++++ core/repair_jobs/canonical_version_resolve.py | 24 ++++++++++++++++++- tests/test_canonical_orchestration.py | 5 ++++ tests/test_canonical_version_job.py | 15 ++++++++++++ 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/core/metadata/canonical_resolver.py b/core/metadata/canonical_resolver.py index b521dcb1..8ce84d40 100644 --- a/core/metadata/canonical_resolver.py +++ b/core/metadata/canonical_resolver.py @@ -115,6 +115,11 @@ def resolve_canonical_for_album( return None detail = score_release_detail(file_tracks, winner['_tracks']) + # Pinned-release track titles — already fetched, so free. Capped so a giant + # box set can't bloat the finding's details_json. + release_titles = [ + (t.get('title') or t.get('name') or '') for t in winner['_tracks'] + ][:60] return { 'source': winner['source'], 'album_id': winner['album_id'], @@ -124,6 +129,7 @@ def resolve_canonical_for_album( 'count_fit': detail['count_fit'], 'duration_fit': detail['duration_fit'], 'title_fit': detail['title_fit'], + 'release_track_titles': release_titles, 'candidates': [ {'source': e['source'], 'album_id': e['album_id'], 'track_count': e['track_count'], 'score': e['score']} @@ -243,6 +249,14 @@ def resolve_and_store_canonical_for_album( # already loaded — no extra query). Storage only uses source/id/score. result['album_title'] = album_data.get('title') or '' result['artist_name'] = album_data.get('artist_name') or '' + # Free context off the album row + the data we already gathered. + if album_data.get('year'): + result['year'] = album_data['year'] + result['db_track_count'] = album_data.get('track_count') or len(file_tracks) + if album_data.get('duration'): + result['db_duration_ms'] = album_data['duration'] + result['linked_sources'] = source_ids # {source: album_id} the album points at now + result['file_track_titles'] = [ft.get('title') or '' for ft in file_tracks][:60] if album_data.get('thumb_url'): result['album_thumb_url'] = album_data['thumb_url'] # Artist thumb via a guarded lookup (not the shared album loader — some diff --git a/core/repair_jobs/canonical_version_resolve.py b/core/repair_jobs/canonical_version_resolve.py index 6c802040..ea7b91a3 100644 --- a/core/repair_jobs/canonical_version_resolve.py +++ b/core/repair_jobs/canonical_version_resolve.py @@ -28,7 +28,14 @@ def _pct(v) -> str: def _describe_pin(resolved: dict) -> str: """Human-readable, judge-able explanation of WHY this release was chosen.""" + artist = resolved.get('artist_name') or '' + album = resolved.get('album_title') or '' + head = f"{artist} — {album}".strip(" —") or resolved.get('album_id', '') + year = resolved.get('year') + if year: + head += f" ({year})" lines = [ + f"{head}" if head else "", f"Pin {resolved['source']} release {resolved['album_id']} " f"(confidence {_pct(resolved.get('score'))}).", f"Fit to your library: {resolved.get('file_track_count', '?')} files vs " @@ -37,6 +44,13 @@ def _describe_pin(resolved: dict) -> str: f"durations {_pct(resolved.get('duration_fit'))}, " f"titles {_pct(resolved.get('title_fit'))}.", ] + + # What the album is currently linked to vs what we'd pin. + linked = resolved.get('linked_sources') or {} + if linked: + linked_str = ", ".join(f"{s}={i}" for s, i in linked.items()) + lines.append(f"Currently linked: {linked_str} → pinning {resolved['source']}.") + others = [c for c in resolved.get('candidates', []) if c.get('source') != resolved.get('source')] if others: comp = ", ".join( @@ -45,7 +59,15 @@ def _describe_pin(resolved: dict) -> str: lines.append(f"Beat: {comp}.") elif len(resolved.get('candidates', [])) == 1: lines.append("Only this source had a release linked for this album.") - return "\n".join(lines) + + # Track listing of the pinned release (so you can eyeball the actual songs). + titles = resolved.get('release_track_titles') or [] + if titles: + shown = "; ".join(f"{i+1}. {t}" for i, t in enumerate(titles[:25])) + more = f" (+{len(titles) - 25} more)" if len(titles) > 25 else "" + lines.append(f"Release tracks: {shown}{more}") + + return "\n".join(l for l in lines if l) @register_job diff --git a/tests/test_canonical_orchestration.py b/tests/test_canonical_orchestration.py index 931302a3..ec933bb1 100644 --- a/tests/test_canonical_orchestration.py +++ b/tests/test_canonical_orchestration.py @@ -94,6 +94,11 @@ def test_result_includes_artist_and_album_context(tmp_path): assert out["artist_name"] == "Imagine Dragons" assert out["album_thumb_url"] == "http://album.jpg" assert out["artist_thumb_url"] == "http://artist.jpg" + # free context: db track count, linked sources, and both title lists + assert out["db_track_count"] == 11 + assert out["linked_sources"] == {"spotify": "sp1"} + assert out["file_track_titles"][0] == "Song 1" and len(out["file_track_titles"]) == 11 + assert "Song 1" in out["release_track_titles"] def test_resolve_returns_none_when_album_has_no_source_ids(tmp_path): diff --git a/tests/test_canonical_version_job.py b/tests/test_canonical_version_job.py index 9e720adc..f033ecb8 100644 --- a/tests/test_canonical_version_job.py +++ b/tests/test_canonical_version_job.py @@ -75,6 +75,21 @@ def test_describe_pin_is_judgeable(): assert "Beat:" in desc and "spotify 65% (17 tk)" in desc # what it beat +def test_describe_pin_includes_year_linked_and_tracklist(): + desc = _describe_pin({ + "source": "deezer", "album_id": "dz1", "score": 1.0, + "artist_name": "Lenka", "album_title": "Souls of Serenity", "year": 2017, + "file_track_count": 3, "release_track_count": 3, + "count_fit": 1.0, "duration_fit": 1.0, "title_fit": 1.0, + "linked_sources": {"spotify": "sp1", "deezer": "dz1"}, + "release_track_titles": ["The Show", "Trouble Is a Friend", "Everything at Once"], + "candidates": [{"source": "deezer", "album_id": "dz1", "track_count": 3, "score": 1.0}], + }) + assert "Lenka — Souls of Serenity (2017)" in desc + assert "Currently linked: spotify=sp1, deezer=dz1 → pinning deezer" in desc + assert "Release tracks: 1. The Show; 2. Trouble Is a Friend; 3. Everything at Once" in desc + + def test_describe_pin_single_source(): desc = _describe_pin({ "source": "spotify", "album_id": "x", "score": 0.9, From dfa5204e0a27c82ec154203c0d430290a34c93a3 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 15:33:13 -0700 Subject: [PATCH 020/108] Repair settings: dropdown for fixed-choice settings (canonical source_selection) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The canonical source_selection setting was rendering as a free-text box — easy to typo an invalid mode. Added a generic choice mechanism so it's a dropdown: - RepairJob.setting_options: {key: [allowed values]} (default {} — opt-in). - CanonicalVersionResolveJob declares source_selection's three modes. - repair_worker.get_all_job_info() includes setting_options in the job payload. - enrichment.js renders a .value as a string, so no change needed there. Generic — any future job can get dropdowns the same way. Jobs that don't declare setting_options are untouched (empty dict -> existing input rendering). Tests: source_selection exposes the 3 options and its default is one of them. 23 repair-job/worker + canonical tests pass (other jobs unaffected). --- core/repair_jobs/base.py | 3 +++ core/repair_jobs/canonical_version_resolve.py | 4 ++++ core/repair_worker.py | 3 +++ tests/test_canonical_version_job.py | 8 ++++++++ webui/static/enrichment.js | 12 ++++++++++++ 5 files changed, 30 insertions(+) diff --git a/core/repair_jobs/base.py b/core/repair_jobs/base.py index 4ac7f52b..d0564545 100644 --- a/core/repair_jobs/base.py +++ b/core/repair_jobs/base.py @@ -100,6 +100,9 @@ class RepairJob(ABC): default_enabled: bool = False default_interval_hours: int = 24 default_settings: Dict[str, Any] = {} + # Optional {setting_key: [allowed values]} — the UI renders a dropdown for + # these instead of a free-text box. Keys not listed render by value type. + setting_options: Dict[str, list] = {} auto_fix: bool = False @abstractmethod diff --git a/core/repair_jobs/canonical_version_resolve.py b/core/repair_jobs/canonical_version_resolve.py index ea7b91a3..61b28655 100644 --- a/core/repair_jobs/canonical_version_resolve.py +++ b/core/repair_jobs/canonical_version_resolve.py @@ -101,6 +101,10 @@ class CanonicalVersionResolveJob(RepairJob): # source matches the files best, regardless of which it is). 'source_selection': 'active_preferred', } + # Render source_selection as a dropdown (not a text box) in the settings UI. + setting_options = { + 'source_selection': ['active_preferred', 'active_only', 'best_fit'], + } auto_fix = True def _get_settings(self, context: JobContext) -> dict: diff --git a/core/repair_worker.py b/core/repair_worker.py index 87ba8abb..289ad3cd 100644 --- a/core/repair_worker.py +++ b/core/repair_worker.py @@ -371,6 +371,9 @@ class RepairWorker: 'interval_hours': config['interval_hours'], 'settings': config['settings'], 'default_settings': job.default_settings.copy(), + # Per-setting choice lists so the UI can render a dropdown + # instead of a free-text box (e.g. canonical source_selection). + 'setting_options': dict(getattr(job, 'setting_options', {}) or {}), 'last_run': last_run, 'next_run': next_run, 'is_running': self._current_job_id == job_id, diff --git a/tests/test_canonical_version_job.py b/tests/test_canonical_version_job.py index f033ecb8..075fa461 100644 --- a/tests/test_canonical_version_job.py +++ b/tests/test_canonical_version_job.py @@ -59,6 +59,14 @@ def test_source_selection_defaults_to_active_preferred(): assert CanonicalVersionResolveJob.default_settings["source_selection"] == "active_preferred" +def test_source_selection_exposes_dropdown_options(): + # The UI renders a ${optionsHtml} + `; + } const inputType = typeof val === 'boolean' ? 'checkbox' : typeof val === 'number' ? 'number' : 'text'; const inputVal = inputType === 'checkbox' ? From eaf74732f943cb4e4bb662ead932524460eccd70 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 15:42:14 -0700 Subject: [PATCH 021/108] Canonical: fix ruff lint (B023 loop-bound lambda, S110 bare except-pass) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - B023: default_fetch_tracklist built a per-item lambda closing over the loop variable `it`. Replaced with a module-level _item_get(item, key, default) helper (takes the item as a param — no closure). Behavior unchanged; the dict/object normalization test still passes. - S110: the two best-effort guards in the canonical job (skip-already-pinned read, estimate_scope active-server read) now carry `# noqa: S110 — `, matching the repo's existing convention for intentional swallow-and-continue. ruff check passes on all canonical files + tests; 30 affected tests green. --- core/metadata/canonical_resolver.py | 14 +++++++++----- core/repair_jobs/canonical_version_resolve.py | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/core/metadata/canonical_resolver.py b/core/metadata/canonical_resolver.py index 8ce84d40..a18b4b8b 100644 --- a/core/metadata/canonical_resolver.py +++ b/core/metadata/canonical_resolver.py @@ -138,6 +138,11 @@ def resolve_canonical_for_album( } +def _item_get(item: Any, key: str, default: Any = None) -> Any: + """Read ``key`` from a track item that may be a dict or an object.""" + return item.get(key, default) if isinstance(item, dict) else getattr(item, key, default) + + def default_fetch_tracklist(source: str, album_id: str) -> Optional[List[Dict[str, Any]]]: """Production ``fetch_tracklist``: pull a release's tracklist from a metadata source and normalise to ``{title, track_number, duration_ms}``. Duration is @@ -155,14 +160,13 @@ def default_fetch_tracklist(source: str, album_id: str) -> Optional[List[Dict[st items = items.get('items') or [] out: List[Dict[str, Any]] = [] for it in items: - get = it.get if isinstance(it, dict) else (lambda k, d=None: getattr(it, k, d)) - dur = get('duration_ms') + dur = _item_get(it, 'duration_ms') if dur is None: - secs = get('duration') # some sources give seconds + secs = _item_get(it, 'duration') # some sources give seconds dur = int(secs * 1000) if isinstance(secs, (int, float)) and secs else None out.append({ - 'title': get('name') or get('title') or '', - 'track_number': get('track_number'), + 'title': _item_get(it, 'name') or _item_get(it, 'title') or '', + 'track_number': _item_get(it, 'track_number'), 'duration_ms': dur, }) return out or None diff --git a/core/repair_jobs/canonical_version_resolve.py b/core/repair_jobs/canonical_version_resolve.py index 61b28655..f7fd2f00 100644 --- a/core/repair_jobs/canonical_version_resolve.py +++ b/core/repair_jobs/canonical_version_resolve.py @@ -168,7 +168,7 @@ class CanonicalVersionResolveJob(RepairJob): result.skipped += 1 result.scanned += 1 continue - except Exception: + except Exception: # noqa: S110 — best-effort skip check; on read error just resolve it pass try: @@ -216,6 +216,6 @@ class CanonicalVersionResolveJob(RepairJob): if context.config_manager: try: active_server = context.config_manager.get_active_media_server() - except Exception: + except Exception: # noqa: S110 — best-effort; fall back to no server filter pass return len(self._load_album_ids(context.db, active_server)) From 7956aaac9e84d8f868b3aeb513068100c91cb9f0 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 16:02:26 -0700 Subject: [PATCH 022/108] Fix #772: manual import progress bar stuck at 0 / 'Failed' on slow imports Per-track import does heavy synchronous server-side enrichment (metadata, art, lyrics) that can take 60-90s/track, far longer when external sources are degraded. The React apiClient (ky) had no timeout, so ky's default 10s aborted the import-process request client-side even though the server completed the import (200) and moved the files. The import loop then counted the aborted call as an error, so the bar stayed at 0 and flipped to 'Failed' while files imported fine. Give the two import-process calls (album/process, singles/process) an explicit 5-min timeout. Scoped to import only -- every other endpoint keeps the 10s default; bounded, not disabled. Server behavior unchanged. Adds a test asserting both calls pass the long timeout. --- webui/src/routes/import/-import.api.test.ts | 27 +++++++++++++++++++-- webui/src/routes/import/-import.api.ts | 11 +++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/webui/src/routes/import/-import.api.test.ts b/webui/src/routes/import/-import.api.test.ts index 1963b1f7..c8aaa83d 100644 --- a/webui/src/routes/import/-import.api.test.ts +++ b/webui/src/routes/import/-import.api.test.ts @@ -1,8 +1,14 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; +import { apiClient } from '@/app/api-client'; import { HttpResponse, http, server } from '@/test/msw'; -import { approveAutoImportResult, rejectAutoImportResult } from './-import.api'; +import { + approveAutoImportResult, + processImportAlbumTrack, + processImportSingleFile, + rejectAutoImportResult, +} from './-import.api'; const softFailureMessage = 'Item not found or not pending review'; @@ -26,4 +32,21 @@ describe('import api', () => { await expect(approveAutoImportResult(17)).rejects.toThrow(softFailureMessage); await expect(rejectAutoImportResult(18)).rejects.toThrow(softFailureMessage); }); + + it('#772: import-process calls use a long timeout, not ky default 10s', async () => { + // Per-track import does heavy server-side enrichment (60-90s+); the default + // 10s timeout aborted it client-side -> progress bar stuck + "Failed" while + // files imported. These calls must pass an explicit long timeout. + const ok = { json: async () => ({ success: true, processed: 1, total: 1, errors: [] }) }; + const spy = vi.spyOn(apiClient, 'post').mockReturnValue(ok as never); + + await processImportAlbumTrack({ album: {} as never, match: {} as never }); + await processImportSingleFile({}); + + expect(spy).toHaveBeenCalledTimes(2); + for (const call of spy.mock.calls) { + expect(call[1]).toMatchObject({ timeout: 300_000 }); + } + spy.mockRestore(); + }); }); diff --git a/webui/src/routes/import/-import.api.ts b/webui/src/routes/import/-import.api.ts index 8de7234d..00de11e2 100644 --- a/webui/src/routes/import/-import.api.ts +++ b/webui/src/routes/import/-import.api.ts @@ -18,6 +18,15 @@ import type { export const IMPORT_QUERY_KEY = ['import'] as const; +// Per-track import does heavy synchronous enrichment server-side (metadata +// lookups, art, lyrics) and can legitimately take 60-90s/track — much longer +// when external sources are degraded. ky's default 10s timeout aborts those +// requests client-side even though the server completes the import (200), +// which left the progress bar stuck at 0 and showing "Failed" while files +// imported fine (#772). Give the import-process calls a generous bound so the +// responses actually arrive and the bar advances. Scoped to import only. +const IMPORT_REQUEST_TIMEOUT_MS = 300_000; // 5 min/track + export async function fetchImportStagingFiles(): Promise { return readJson(apiClient.get('import/staging/files')); } @@ -71,6 +80,7 @@ export async function processImportAlbumTrack(input: { album: input.album, matches: [input.match], }, + timeout: IMPORT_REQUEST_TIMEOUT_MS, }), ); } @@ -92,6 +102,7 @@ export async function processImportSingleFile(file: unknown): Promise Date: Tue, 2 Jun 2026 19:06:44 -0700 Subject: [PATCH 023/108] Add Manage Enrichment Workers modal (v1 + polish) Dashboard 'enrichment bubbles' could pause/hover but offered no way to *manage* a worker. This adds a full management modal opened from a new header button, covering all 11 enrichment sources. Backend (testable core helper + seam tests; no live-DB dependency): - core/enrichment/unmatched.py: pure, whitelisted SQL builders for the unmatched browser. service/entity validated against a support map (never interpolated raw); search + pagination bound as params; tracks join albums for artwork; limit capped at 200. - database/music_database.py: get_enrichment_unmatched() + get_enrichment_breakdown() (the breakdown splits matched/not_found/pending, which the existing get_stats().progress lumps together). - core/enrichment/api.py: GET /api/enrichment//{unmatched,breakdown} on the existing blueprint + a db_getter hook. - web_server.py: wire db_getter=get_database. - tests/enrichment/test_unmatched.py: 19 tests across builders, DB methods, and Flask routes. Frontend (vanilla, matches app conventions): - webui/static/enrichment-manager.js: worker rail with live status + coverage micro-bars, accent-themed detail panel (hero header, segmented matched/ not_found/pending stat cards, current item, pause/resume), and a searchable paginated unmatched browser with inline manual match (reusing search-service + manual-match) and retry (clear-match re-queues). - Polish: entrance/exit motion, scroll-lock, Escape, refresh control, flicker-free polling (in-place updates), skeleton loaders, relative timestamps, per-worker accent theming, real dashboard logos reused at runtime (with the same invert/circle treatment), responsive rail. - index.html: header button + script include. style.css: full styling. Reuses existing pause/resume, status, and manual search+assign endpoints. Backend tests green (19 new + 11 existing enrichment tests). --- core/enrichment/api.py | 76 ++- core/enrichment/unmatched.py | 214 ++++++++ database/music_database.py | 58 ++ tests/enrichment/test_unmatched.py | 194 +++++++ web_server.py | 1 + webui/index.html | 8 + webui/static/enrichment-manager.js | 816 +++++++++++++++++++++++++++++ webui/static/style.css | 385 ++++++++++++++ 8 files changed, 1749 insertions(+), 3 deletions(-) create mode 100644 core/enrichment/unmatched.py create mode 100644 tests/enrichment/test_unmatched.py create mode 100644 webui/static/enrichment-manager.js diff --git a/core/enrichment/api.py b/core/enrichment/api.py index e4d097d7..7cf472f7 100644 --- a/core/enrichment/api.py +++ b/core/enrichment/api.py @@ -18,9 +18,14 @@ from __future__ import annotations from typing import Any, Callable, Optional -from flask import Blueprint, jsonify +from flask import Blueprint, jsonify, request from core.enrichment.services import EnrichmentService, get_service +from core.enrichment.unmatched import ( + SERVICE_ENTITY_SUPPORT, + UnmatchedQueryError, + supported_entity_types, +) from utils.logging_config import get_logger @@ -32,6 +37,7 @@ logger = get_logger("enrichment.api") _config_set: Optional[Callable[[str, Any], None]] = None _auto_paused_discard: Optional[Callable[[str], None]] = None _yield_override_add: Optional[Callable[[str], None]] = None +_db_getter: Optional[Callable[[], Any]] = None def configure( @@ -39,16 +45,19 @@ def configure( config_set: Optional[Callable[[str, Any], None]] = None, auto_paused_discard: Optional[Callable[[str], None]] = None, yield_override_add: Optional[Callable[[str], None]] = None, + db_getter: Optional[Callable[[], Any]] = None, ) -> None: """Wire host-side mutators that the generic routes call after pause/resume. Each is optional — pass None for hosts that don't have a corresponding - mechanism (e.g. tests). + mechanism (e.g. tests). ``db_getter`` returns the live ``MusicDatabase`` + for the unmatched-browser routes. """ - global _config_set, _auto_paused_discard, _yield_override_add + global _config_set, _auto_paused_discard, _yield_override_add, _db_getter _config_set = config_set _auto_paused_discard = auto_paused_discard _yield_override_add = yield_override_add + _db_getter = db_getter def _persist_paused(service: EnrichmentService, paused: bool) -> None: @@ -153,4 +162,65 @@ def create_blueprint() -> Blueprint: logger.error("Error resuming %s worker: %s", service.id, e) return jsonify({'error': str(e)}), 500 + @bp.route('/api/enrichment//breakdown', methods=['GET']) + def enrichment_breakdown(service_id: str): + """matched / not_found / pending tallies per entity type for the modal.""" + if service_id not in SERVICE_ENTITY_SUPPORT: + return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404 + if _db_getter is None: + return jsonify({'error': 'database unavailable'}), 503 + try: + db = _db_getter() + breakdown = { + entity: db.get_enrichment_breakdown(service_id, entity) + for entity in supported_entity_types(service_id) + } + return jsonify({'service': service_id, 'breakdown': breakdown}), 200 + except UnmatchedQueryError as e: + return jsonify({'error': str(e)}), 400 + except Exception as e: + logger.error("Error building %s enrichment breakdown: %s", service_id, e) + return jsonify({'error': str(e)}), 500 + + @bp.route('/api/enrichment//unmatched', methods=['GET']) + def enrichment_unmatched(service_id: str): + """Paginated list of items this source hasn't matched (for manual match). + + Query params: ``entity_type`` (artist|album|track), ``status`` + (not_found|pending|unmatched), ``q`` (name search), ``limit``, ``offset``. + """ + if service_id not in SERVICE_ENTITY_SUPPORT: + return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404 + if _db_getter is None: + return jsonify({'error': 'database unavailable'}), 503 + + entity_type = (request.args.get('entity_type') or 'artist').strip() + status = (request.args.get('status') or 'not_found').strip() + query = (request.args.get('q') or '').strip() or None + try: + limit = int(request.args.get('limit', 50)) + offset = int(request.args.get('offset', 0)) + except (TypeError, ValueError): + return jsonify({'error': 'limit/offset must be integers'}), 400 + + try: + result = _db_getter().get_enrichment_unmatched( + service_id, entity_type, status, query, limit, offset + ) + except UnmatchedQueryError as e: + return jsonify({'error': str(e)}), 400 + except Exception as e: + logger.error("Error listing %s unmatched %ss: %s", service_id, entity_type, e) + return jsonify({'error': str(e)}), 500 + + result.update({ + 'service': service_id, + 'entity_type': entity_type, + 'status': status, + 'limit': limit, + 'offset': offset, + 'entity_types': list(supported_entity_types(service_id)), + }) + return jsonify(result), 200 + return bp diff --git a/core/enrichment/unmatched.py b/core/enrichment/unmatched.py new file mode 100644 index 00000000..924a3bd9 --- /dev/null +++ b/core/enrichment/unmatched.py @@ -0,0 +1,214 @@ +"""Read-side helpers for browsing the items an enrichment source hasn't matched. + +The dashboard "Manage Enrichment Workers" modal lists, per source, the +artists / albums / tracks whose ``_match_status`` is ``'not_found'`` +(or still pending = ``NULL``) so the user can manually match them. Every +enrichment source writes a uniform ``_match_status`` column, so one +parametric query serves all 11 workers. + +This module owns the column mapping and SQL construction. ``service`` and +``entity_type`` are whitelisted against :data:`SERVICE_ENTITY_SUPPORT` and the +entity table map before any column name is interpolated — user-supplied values +(the search term, pagination) are always bound parameters, never interpolated. +""" + +from __future__ import annotations + +from typing import List, Optional, Tuple + +# Which entity types each enrichment source covers. Mirrors the authoritative +# ``_SERVICE_ID_COLUMNS`` map in web_server.py (used by manual-match), kept here +# so the unmatched browser is self-contained and unit-testable. Singular keys +# ('artist'/'album'/'track') match the manual-match entity_type vocabulary. +SERVICE_ENTITY_SUPPORT = { + 'spotify': ('artist', 'album', 'track'), + 'musicbrainz': ('artist', 'album', 'track'), + 'deezer': ('artist', 'album', 'track'), + 'audiodb': ('artist', 'album', 'track'), + 'discogs': ('artist', 'album'), # no track-level id column + 'itunes': ('artist', 'album', 'track'), + 'lastfm': ('artist', 'album', 'track'), + 'genius': ('artist', 'track'), # no album-level id column + 'tidal': ('artist', 'album', 'track'), + 'qobuz': ('artist', 'album', 'track'), + 'amazon': ('artist', 'album', 'track'), +} + +# entity_type -> table / display-name column / image expression / optional join. +# tracks carry no artwork column of their own, so we borrow the parent album's. +_ENTITY_TABLE = { + 'artist': { + 'table': 'artists', 'name': 'name', + 'image': 'artists.thumb_url', 'join': '', + }, + 'album': { + 'table': 'albums', 'name': 'title', + 'image': 'albums.thumb_url', 'join': '', + }, + 'track': { + 'table': 'tracks', 'name': 'title', + 'image': 'al.thumb_url', + 'join': 'LEFT JOIN albums al ON tracks.album_id = al.id', + }, +} + +# 'unmatched' = not yet matched at all (pending OR explicitly not_found). +VALID_STATUSES = ('not_found', 'pending', 'unmatched') + +# Hard cap so a malicious/buggy caller can't ask for the whole library at once. +MAX_LIMIT = 200 + + +class UnmatchedQueryError(ValueError): + """Raised for an unknown service / unsupported entity type / bad status.""" + + +def supported_entity_types(service: str) -> Tuple[str, ...]: + """Return the entity types a source enriches, or () for an unknown source.""" + return SERVICE_ENTITY_SUPPORT.get(service, ()) + + +def match_status_column(service: str) -> str: + return f"{service}_match_status" + + +def last_attempted_column(service: str) -> str: + return f"{service}_last_attempted" + + +def _validate(service: str, entity_type: str) -> None: + support = SERVICE_ENTITY_SUPPORT.get(service) + if support is None: + raise UnmatchedQueryError(f"Unknown enrichment service: {service!r}") + if entity_type not in support: + raise UnmatchedQueryError( + f"{service} does not enrich {entity_type!r} entities" + ) + if entity_type not in _ENTITY_TABLE: # defensive — support map drift + raise UnmatchedQueryError(f"No table mapping for entity type {entity_type!r}") + + +def _status_predicate(service: str, status: str, qualifier: str) -> str: + """SQL predicate selecting rows in the requested match state. + + ``qualifier`` (the table name/alias) is always prefixed so the predicate is + unambiguous even when the query joins a second table that also carries a + ``_match_status`` column (tracks LEFT JOIN albums). + """ + col = f"{qualifier}.{match_status_column(service)}" + if status == 'not_found': + return f"{col} = 'not_found'" + if status == 'pending': + return f"{col} IS NULL" + # 'unmatched' + return f"({col} IS NULL OR {col} = 'not_found')" + + +def build_unmatched_query( + service: str, + entity_type: str, + status: str = 'not_found', + query: Optional[str] = None, + limit: int = 50, + offset: int = 0, +) -> Tuple[str, List]: + """Build the paginated SELECT for one (service, entity_type, status) view. + + Returns ``(sql, params)``. Selected columns: id, name, image_url, status, + last_attempted. + """ + _validate(service, entity_type) + if status not in VALID_STATUSES: + raise UnmatchedQueryError(f"Invalid status: {status!r}") + + meta = _ENTITY_TABLE[entity_type] + table, name_col, image_expr, join = ( + meta['table'], meta['name'], meta['image'], meta['join'], + ) + ms = match_status_column(service) + la = last_attempted_column(service) + + where = [_status_predicate(service, status, table)] + params: List = [] + if query: + where.append(f"{table}.{name_col} LIKE ?") + params.append(f"%{query}%") + + sql = ( + f"SELECT {table}.id AS id, {table}.{name_col} AS name, " + f"{image_expr} AS image_url, {table}.{ms} AS status, " + f"{table}.{la} AS last_attempted " + f"FROM {table} {join} " + f"WHERE {' AND '.join(where)} " + f"ORDER BY {table}.{name_col} COLLATE NOCASE " + f"LIMIT ? OFFSET ?" + ).replace(' ', ' ') + + params.append(_clamp_limit(limit)) + params.append(max(int(offset or 0), 0)) + return sql, params + + +def build_count_query( + service: str, + entity_type: str, + status: str = 'not_found', + query: Optional[str] = None, +) -> Tuple[str, List]: + """Build the COUNT(*) matching :func:`build_unmatched_query`'s filters.""" + _validate(service, entity_type) + if status not in VALID_STATUSES: + raise UnmatchedQueryError(f"Invalid status: {status!r}") + + meta = _ENTITY_TABLE[entity_type] + table, name_col = meta['table'], meta['name'] + + where = [_status_predicate(service, status, table)] + params: List = [] + if query: + where.append(f"{table}.{name_col} LIKE ?") + params.append(f"%{query}%") + + sql = f"SELECT COUNT(*) FROM {table} WHERE {' AND '.join(where)}" + return sql, params + + +def build_breakdown_query(service: str, entity_type: str) -> Tuple[str, List]: + """Build the matched / not_found / pending / total tally for one entity type.""" + _validate(service, entity_type) + meta = _ENTITY_TABLE[entity_type] + table = meta['table'] + ms = f"{table}.{match_status_column(service)}" + sql = ( + "SELECT " + f"SUM(CASE WHEN {ms} = 'matched' THEN 1 ELSE 0 END) AS matched, " + f"SUM(CASE WHEN {ms} = 'not_found' THEN 1 ELSE 0 END) AS not_found, " + f"SUM(CASE WHEN {ms} IS NULL THEN 1 ELSE 0 END) AS pending, " + f"COUNT(*) AS total " + f"FROM {table}" + ) + return sql, [] + + +def _clamp_limit(limit) -> int: + try: + n = int(limit) + except (TypeError, ValueError): + return 50 + if n <= 0: + return 50 + return min(n, MAX_LIMIT) + + +__all__ = [ + 'SERVICE_ENTITY_SUPPORT', + 'VALID_STATUSES', + 'MAX_LIMIT', + 'UnmatchedQueryError', + 'supported_entity_types', + 'match_status_column', + 'last_attempted_column', + 'build_unmatched_query', + 'build_count_query', + 'build_breakdown_query', +] diff --git a/database/music_database.py b/database/music_database.py index 74fa5e13..7dee6199 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -1021,6 +1021,64 @@ class MusicDatabase: finally: conn.close() + def get_enrichment_unmatched( + self, + service: str, + entity_type: str, + status: str = 'not_found', + query: str = None, + limit: int = 50, + offset: int = 0, + ) -> dict: + """List items a given enrichment source hasn't matched, paginated. + + Powers the "Manage Enrichment Workers" modal's unmatched browser. + Returns ``{'total': int, 'items': [{id, name, image_url, status, + last_attempted}]}``. Raises ``UnmatchedQueryError`` for an unknown + service / unsupported entity type / bad status (the caller maps that to + an HTTP 400).""" + from core.enrichment.unmatched import ( + build_count_query, + build_unmatched_query, + ) + + sql, params = build_unmatched_query( + service, entity_type, status, query, limit, offset + ) + count_sql, count_params = build_count_query(service, entity_type, status, query) + conn = self._get_connection() + try: + cursor = conn.cursor() + total = cursor.execute(count_sql, count_params).fetchone()[0] + rows = cursor.execute(sql, params).fetchall() + items = [dict(row) for row in rows] + return {'total': total or 0, 'items': items} + finally: + conn.close() + + def get_enrichment_breakdown(self, service: str, entity_type: str) -> dict: + """Return ``{matched, not_found, pending, total}`` for a source/entity. + + The per-worker ``get_stats().progress`` lumps matched + not_found into a + single 'processed' count; this splits them so the modal can show the + real match rate. Raises ``UnmatchedQueryError`` on bad input.""" + from core.enrichment.unmatched import build_breakdown_query + + sql, params = build_breakdown_query(service, entity_type) + conn = self._get_connection() + try: + row = conn.cursor().execute(sql, params).fetchone() + if not row: + return {'matched': 0, 'not_found': 0, 'pending': 0, 'total': 0} + return { + 'matched': row[0] or 0, + 'not_found': row[1] or 0, + 'pending': row[2] or 0, + 'total': row[3] or 0, + } + finally: + conn.close() + def _add_mirrored_playlist_explored_column(self, cursor): """Add explored_at column to mirrored_playlists to persist explore badge.""" try: diff --git a/tests/enrichment/test_unmatched.py b/tests/enrichment/test_unmatched.py new file mode 100644 index 00000000..0a0730ab --- /dev/null +++ b/tests/enrichment/test_unmatched.py @@ -0,0 +1,194 @@ +"""Unmatched-browser backend for the Manage Enrichment Workers modal. + +Three seams: + * pure SQL builders + validation (core.enrichment.unmatched) + * the MusicDatabase read methods against a temp DB + * the Flask routes via a test client +""" + +from __future__ import annotations + +import pytest +from flask import Flask + +from core.enrichment import api as enrichment_api +from core.enrichment.unmatched import ( + MAX_LIMIT, + UnmatchedQueryError, + build_breakdown_query, + build_count_query, + build_unmatched_query, + supported_entity_types, +) +from database.music_database import MusicDatabase + + +# -------------------------------------------------------------------------- +# Pure builders / validation +# -------------------------------------------------------------------------- + +def test_unknown_service_rejected(): + with pytest.raises(UnmatchedQueryError): + build_unmatched_query('not-a-service', 'artist') + + +def test_unsupported_entity_type_rejected(): + # Genius enriches artists + tracks but has no album-level id column. + assert 'album' not in supported_entity_types('genius') + with pytest.raises(UnmatchedQueryError): + build_unmatched_query('genius', 'album') + with pytest.raises(UnmatchedQueryError): + build_breakdown_query('discogs', 'track') # discogs has no track column + + +def test_bad_status_rejected(): + with pytest.raises(UnmatchedQueryError): + build_unmatched_query('spotify', 'artist', status='bogus') + + +def test_status_predicates(): + nf, _ = build_count_query('spotify', 'artist', 'not_found') + pend, _ = build_count_query('spotify', 'artist', 'pending') + un, _ = build_count_query('spotify', 'artist', 'unmatched') + assert "artists.spotify_match_status = 'not_found'" in nf + assert "artists.spotify_match_status IS NULL" in pend + assert "IS NULL OR" in un and "= 'not_found'" in un + + +def test_track_query_qualifies_status_to_avoid_join_ambiguity(): + # tracks LEFT JOIN albums for artwork — both carry spotify_match_status, + # so the predicate must be qualified or SQLite errors "ambiguous column". + sql, _ = build_unmatched_query('spotify', 'track', 'not_found') + assert 'LEFT JOIN albums al' in sql + assert 'tracks.spotify_match_status' in sql + assert 'al.thumb_url AS image_url' in sql + + +def test_search_adds_like_param(): + sql, params = build_unmatched_query('spotify', 'artist', 'not_found', query='dragons') + assert 'LIKE ?' in sql + assert '%dragons%' in params + + +def test_limit_is_clamped(): + _, params = build_unmatched_query('spotify', 'artist', 'not_found', limit=99999) + assert params[-2] == MAX_LIMIT # limit + assert params[-1] == 0 # offset + _, params2 = build_unmatched_query('spotify', 'artist', 'not_found', limit=0) + assert params2[-2] == 50 # invalid -> default + + +# -------------------------------------------------------------------------- +# MusicDatabase integration (temp DB) +# -------------------------------------------------------------------------- + +def _seed(db: MusicDatabase): + conn = db._get_connection() + cur = conn.cursor() + # 3 artists: matched / not_found / pending(NULL) + cur.execute("INSERT INTO artists (id, name, spotify_match_status) VALUES ('a1','Matched Artist','matched')") + cur.execute("INSERT INTO artists (id, name, spotify_match_status) VALUES ('a2','Failed Dragons','not_found')") + cur.execute("INSERT INTO artists (id, name) VALUES ('a3','Pending Person')") # NULL status + # album + track to exercise the join-for-artwork path + cur.execute("INSERT INTO albums (id, artist_id, title, thumb_url, spotify_match_status) " + "VALUES ('al1','a2','Evolve','http://img/evolve.jpg','not_found')") + cur.execute("INSERT INTO tracks (id, album_id, artist_id, title, spotify_match_status) " + "VALUES ('t1','al1','a2','Believer','not_found')") + conn.commit() + conn.close() + + +@pytest.fixture +def db(tmp_path): + d = MusicDatabase(str(tmp_path / 'enrich.db')) + _seed(d) + return d + + +def test_breakdown_splits_matched_notfound_pending(db): + bd = db.get_enrichment_breakdown('spotify', 'artist') + assert bd == {'matched': 1, 'not_found': 1, 'pending': 1, 'total': 3} + + +def test_unmatched_not_found_only(db): + res = db.get_enrichment_unmatched('spotify', 'artist', status='not_found') + assert res['total'] == 1 + assert [i['name'] for i in res['items']] == ['Failed Dragons'] + assert res['items'][0]['status'] == 'not_found' + + +def test_unmatched_pending_only(db): + res = db.get_enrichment_unmatched('spotify', 'artist', status='pending') + assert res['total'] == 1 + assert res['items'][0]['name'] == 'Pending Person' + + +def test_unmatched_combined(db): + res = db.get_enrichment_unmatched('spotify', 'artist', status='unmatched') + assert res['total'] == 2 + assert {i['name'] for i in res['items']} == {'Failed Dragons', 'Pending Person'} + + +def test_unmatched_search_filters_by_name(db): + res = db.get_enrichment_unmatched('spotify', 'artist', status='unmatched', query='dragons') + assert res['total'] == 1 + assert res['items'][0]['name'] == 'Failed Dragons' + + +def test_unmatched_pagination(db): + page = db.get_enrichment_unmatched('spotify', 'artist', status='unmatched', limit=1, offset=0) + assert page['total'] == 2 and len(page['items']) == 1 + page2 = db.get_enrichment_unmatched('spotify', 'artist', status='unmatched', limit=1, offset=1) + assert page2['items'][0]['name'] != page['items'][0]['name'] + + +def test_track_unmatched_borrows_album_artwork(db): + res = db.get_enrichment_unmatched('spotify', 'track', status='not_found') + assert res['total'] == 1 + assert res['items'][0]['name'] == 'Believer' + assert res['items'][0]['image_url'] == 'http://img/evolve.jpg' + + +def test_db_raises_on_bad_input(db): + with pytest.raises(UnmatchedQueryError): + db.get_enrichment_unmatched('spotify', 'artist', status='bogus') + + +# -------------------------------------------------------------------------- +# Flask routes +# -------------------------------------------------------------------------- + +@pytest.fixture +def client(db): + enrichment_api.configure(db_getter=lambda: db) + app = Flask(__name__) + app.register_blueprint(enrichment_api.create_blueprint()) + with app.test_client() as c: + yield c + enrichment_api.configure(db_getter=None) # reset module global + + +def test_route_unknown_service_404(client): + assert client.get('/api/enrichment/bogus/unmatched').status_code == 404 + + +def test_route_bad_entity_type_400(client): + # genius has no album column -> 400, not a 500 + r = client.get('/api/enrichment/genius/unmatched?entity_type=album') + assert r.status_code == 400 + + +def test_route_happy_path(client): + r = client.get('/api/enrichment/spotify/unmatched?entity_type=artist&status=unmatched') + assert r.status_code == 200 + body = r.get_json() + assert body['total'] == 2 + assert body['service'] == 'spotify' + assert body['entity_types'] == ['artist', 'album', 'track'] + + +def test_route_breakdown(client): + r = client.get('/api/enrichment/spotify/breakdown') + assert r.status_code == 200 + bd = r.get_json()['breakdown'] + assert bd['artist'] == {'matched': 1, 'not_found': 1, 'pending': 1, 'total': 3} diff --git a/web_server.py b/web_server.py index 2849ac1c..6d0d007e 100644 --- a/web_server.py +++ b/web_server.py @@ -34645,6 +34645,7 @@ _configure_enrichment_api( config_set=lambda key, value: config_manager.set(key, value), auto_paused_discard=lambda token: _download_auto_paused.discard(token), yield_override_add=lambda token: _download_yield_override.add(token), + db_getter=get_database, ) app.register_blueprint(_create_enrichment_blueprint()) diff --git a/webui/index.html b/webui/index.html index 659ba7f4..eb3f5ac6 100644 --- a/webui/index.html +++ b/webui/index.html @@ -625,6 +625,13 @@ + +
@@ -8032,6 +8039,7 @@ + diff --git a/webui/static/enrichment-manager.js b/webui/static/enrichment-manager.js new file mode 100644 index 00000000..f0be70ce --- /dev/null +++ b/webui/static/enrichment-manager.js @@ -0,0 +1,816 @@ +/* + * Manage Enrichment Workers modal. + * + * The dashboard "enrichment bubbles" expose hover/pause but no way to *manage* + * a worker. This modal surfaces, per worker: live status + current item, + * pause/resume, a matched/not-found/pending breakdown per entity type, and a + * searchable/paginated browser of the items that source hasn't matched — each + * with inline manual-match (reusing /api/library/search-service + + * manual-match) and retry (clear-match, which re-queues the item). + * + * Backend: GET /api/enrichment//{status,breakdown,unmatched}, POST + * .../{pause,resume}. The unmatched/breakdown routes are generic across all 11 + * workers (see core/enrichment/unmatched.py). + */ + +// Per-source accent + the CSS selector of that worker's logo already rendered +// in the dashboard bubble. We reuse those exact sources at runtime +// (via _emLogoSrc) so the modal shows the real logos — including AudioDB's +// inline base64 — and stays in sync if the dashboard logos ever change. +// imgFilter / imgRound mirror the per-logo CSS the dashboard bubbles apply, so +// black-on-dark icons (Discogs/Tidal/Qobuz/Amazon) get inverted to white and +// square logos (Last.fm) clip to a circle here too. +const ENRICHMENT_WORKERS = [ + { id: 'spotify', name: 'Spotify', color: '#1db954', logoSel: '.spotify-enrich-logo' }, + { id: 'itunes', name: 'iTunes', color: '#fb5bc5', logoSel: '.itunes-enrich-logo' }, + { id: 'musicbrainz', name: 'MusicBrainz', color: '#ba55d3', logoSel: '.mb-logo' }, + { id: 'deezer', name: 'Deezer', color: '#a238ff', logoSel: '.deezer-logo' }, + { id: 'audiodb', name: 'AudioDB', color: '#1c8cf0', logoSel: '.audiodb-logo' }, + { id: 'discogs', name: 'Discogs', color: '#cfcfcf', logoSel: '.discogs-logo', imgFilter: 'brightness(0) invert(1)' }, + { id: 'lastfm', name: 'Last.fm', color: '#d51007', logoSel: '.lastfm-enrich-logo', imgRound: true }, + { id: 'genius', name: 'Genius', color: '#ffe600', logoSel: '.genius-enrich-logo' }, + { id: 'tidal', name: 'Tidal', color: '#00cfe6', logoSel: '.tidal-enrich-logo', imgFilter: 'invert(1) brightness(1.8)', imgRound: true }, + { id: 'qobuz', name: 'Qobuz', color: '#0070ef', logoSel: '.qobuz-enrich-logo', imgFilter: 'invert(1)', imgRound: true }, + { id: 'amazon', name: 'Amazon Music', color: '#ff9900', logoSel: '.amazon-enrich-logo', imgFilter: 'brightness(0) invert(1)' }, +]; + +const _emWorkerById = Object.fromEntries(ENRICHMENT_WORKERS.map(w => [w.id, w])); + +// '#1db954' -> '29,185,84' for rgba(var(--em-accent-rgb), a) usage. +function _emHexToRgb(hex) { + const h = String(hex || '').replace('#', ''); + const full = h.length === 3 ? h.split('').map(c => c + c).join('') : h; + const n = parseInt(full, 16); + if (isNaN(n) || full.length !== 6) return '120,120,120'; + return `${(n >> 16) & 255},${(n >> 8) & 255},${n & 255}`; +} + +// Resolve a worker's logo URL from the live dashboard bubble (null if absent). +function _emLogoSrc(workerId) { + const w = _emWorkerById[workerId]; + if (!w || !w.logoSel) return null; + const img = document.querySelector(w.logoSel); + return img && img.src ? img.src : null; +} + +// A circular, glowing icon chip mirroring the dashboard bubbles. Falls back to +// a colored initial if the logo is missing or fails to load. +function _emIconHtml(workerId, size) { + const w = _emWorkerById[workerId]; + const src = _emLogoSrc(workerId); + const cls = `em-icon${size === 'lg' ? ' em-icon--lg' : ''}`; + const initial = w.name.charAt(0).toUpperCase(); + const imgStyle = [ + w.imgFilter ? `filter:${w.imgFilter}` : '', + w.imgRound ? 'border-radius:50%' : '', + ].filter(Boolean).join(';'); + const inner = src + ? `` + : `${initial}`; + return `${inner}`; +} + +const enrichmentManagerState = { + open: false, + selected: null, + statuses: {}, // id -> last /status payload + breakdown: null, // selected worker's breakdown + entityTab: 'artist', + statusFilter: 'unmatched', + search: '', + page: 0, + pageSize: 25, + unmatched: null, // { total, items } + pollTimer: null, + loadToken: 0, // guards against out-of-order async renders +}; + +function _emEntityLabel(entity, plural) { + const map = { artist: 'Artist', album: 'Album', track: 'Track' }; + const base = map[entity] || entity; + return plural ? base + 's' : base; +} + +function _emEscape(s) { + return String(s == null ? '' : s).replace(/[&<>"']/g, c => ( + { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c] + )); +} + +// Human "3 days ago" for a SQLite timestamp; '' when never attempted. +function _emRelativeTime(value) { + if (!value) return ''; + // SQLite stores 'YYYY-MM-DD HH:MM:SS' (UTC) — normalize to ISO. + const ts = Date.parse(String(value).replace(' ', 'T') + (String(value).includes('Z') ? '' : 'Z')); + if (isNaN(ts)) return ''; + const secs = Math.max(0, (Date.now() - ts) / 1000); + if (secs < 60) return 'just now'; + const mins = secs / 60; + if (mins < 60) return `${Math.floor(mins)}m ago`; + const hrs = mins / 60; + if (hrs < 24) return `${Math.floor(hrs)}h ago`; + const days = hrs / 24; + if (days < 30) return `${Math.floor(days)}d ago`; + const months = days / 30; + if (months < 12) return `${Math.floor(months)}mo ago`; + return `${Math.floor(months / 12)}y ago`; +} + +// ── Open / close ────────────────────────────────────────────────────────── + +async function openEnrichmentManager() { + let overlay = document.getElementById('enrichment-manager-overlay'); + if (!overlay) { + overlay = document.createElement('div'); + overlay.id = 'enrichment-manager-overlay'; + overlay.className = 'modal-overlay em-overlay hidden'; + overlay.onclick = (e) => { if (e.target === overlay) closeEnrichmentManager(); }; + overlay.innerHTML = ` + `; + document.body.appendChild(overlay); + } + + overlay.classList.remove('hidden', 'em-closing'); + // Re-trigger the entrance animation even when reusing the element. + const modal = overlay.querySelector('.enrichment-manager-modal'); + if (modal) { modal.classList.remove('em-in'); void modal.offsetWidth; modal.classList.add('em-in'); } + document.body.classList.add('em-scroll-lock'); + document.addEventListener('keydown', _emOnKeydown); + enrichmentManagerState.open = true; + + await refreshAllEnrichmentStatuses(); + renderEnrichmentRail(); + + // Default selection: first running worker, else first in the list. + const running = ENRICHMENT_WORKERS.find( + w => enrichmentManagerState.statuses[w.id]?.running + ); + selectEnrichmentWorker((running || ENRICHMENT_WORKERS[0]).id); + if (modal) setTimeout(() => modal.focus(), 60); + + if (enrichmentManagerState.pollTimer) clearInterval(enrichmentManagerState.pollTimer); + enrichmentManagerState.pollTimer = setInterval(_emPollSelected, 3000); +} + +function closeEnrichmentManager() { + const overlay = document.getElementById('enrichment-manager-overlay'); + enrichmentManagerState.open = false; + document.removeEventListener('keydown', _emOnKeydown); + document.body.classList.remove('em-scroll-lock'); + if (enrichmentManagerState.pollTimer) { + clearInterval(enrichmentManagerState.pollTimer); + enrichmentManagerState.pollTimer = null; + } + if (!overlay) return; + // Brief fade/scale-out, then hide. + overlay.classList.add('em-closing'); + setTimeout(() => { + overlay.classList.add('hidden'); + overlay.classList.remove('em-closing'); + }, 170); +} + +// Escape closes the nested match overlay first (if open), else the manager. +function _emOnKeydown(e) { + if (e.key !== 'Escape') return; + const match = document.getElementById('enrichment-match-overlay'); + if (match) { match.remove(); return; } + closeEnrichmentManager(); +} + +// Manual refresh: re-pull every worker's status + the selected worker's data. +async function refreshEnrichmentManager(btn) { + if (btn) btn.classList.add('em-spinning'); + await refreshAllEnrichmentStatuses(); + renderEnrichmentRail(); + const sel = enrichmentManagerState.selected; + if (sel) await Promise.all([_emLoadBreakdown(sel), _emLoadUnmatched()]); + _emRenderStats(); + _emRenderUnmatchedList(); + _emRenderPanelHeader(); + if (btn) setTimeout(() => btn.classList.remove('em-spinning'), 400); +} + +// ── Status loading ────────────────────────────────────────────────────────── + +async function refreshAllEnrichmentStatuses() { + const results = await Promise.all(ENRICHMENT_WORKERS.map(async (w) => { + try { + const res = await fetch(`/api/enrichment/${w.id}/status`); + return [w.id, res.ok ? await res.json() : null]; + } catch (_e) { + return [w.id, null]; + } + })); + for (const [id, status] of results) enrichmentManagerState.statuses[id] = status; +} + +async function _emPollSelected() { + const id = enrichmentManagerState.selected; + if (!id || !enrichmentManagerState.open) return; + try { + const res = await fetch(`/api/enrichment/${id}/status`); + if (res.ok) { + enrichmentManagerState.statuses[id] = await res.json(); + _emUpdateHeaderLive(); // in-place — no logo reflow/flicker + _emUpdateRailRow(id); + } + } catch (_e) { /* transient — keep last */ } +} + +function _emStatusInfo(status) { + if (!status || !status.enabled) return { cls: 'disabled', label: 'Disabled' }; + if (status.rate_limited) return { cls: 'ratelimited', label: 'Rate-limited' }; + if (status.paused) return { cls: 'paused', label: 'Paused' }; + if (status.idle) return { cls: 'idle', label: 'Idle' }; + if (status.running) return { cls: 'running', label: 'Running' }; + return { cls: 'stopped', label: 'Stopped' }; +} + +// ── Left rail ─────────────────────────────────────────────────────────────── + +// Overall library coverage (% of items this source has attempted) from the +// status payload's progress block — a cheap at-a-glance rail signal. +function _emOverallPct(status) { + const p = status && status.progress; + if (!p) return null; + let matched = 0, total = 0; + for (const k of ['artists', 'albums', 'tracks']) { + if (p[k]) { matched += p[k].matched || 0; total += p[k].total || 0; } + } + return total ? Math.round((matched / total) * 100) : 0; +} + +function renderEnrichmentRail() { + const rail = document.getElementById('em-rail'); + if (!rail) return; + rail.innerHTML = ENRICHMENT_WORKERS.map(w => { + const status = enrichmentManagerState.statuses[w.id]; + const info = _emStatusInfo(status); + const pct = _emOverallPct(status); + const cov = pct == null ? '' : ` + `; + return ` + `; + }).join(''); + _emHighlightRail(); +} + +function _emHighlightRail() { + ENRICHMENT_WORKERS.forEach(w => { + const row = document.getElementById(`em-row-${w.id}`); + if (row) row.classList.toggle('active', w.id === enrichmentManagerState.selected); + }); +} + +function _emUpdateRailRow(id) { + const row = document.getElementById(`em-row-${id}`); + if (!row) return; + const status = enrichmentManagerState.statuses[id]; + const info = _emStatusInfo(status); + const pct = _emOverallPct(status); + const dot = row.querySelector('.em-dot'); + if (dot) { dot.className = `em-dot em-dot--${info.cls}`; dot.title = info.label; } + const sub = row.querySelector('.em-worker-sub'); + if (sub) sub.textContent = `${info.label}${pct == null ? '' : ` · ${pct}%`}`; + const cov = row.querySelector('.em-rail-cov-fill'); + if (cov && pct != null) cov.style.width = `${pct}%`; +} + +// ── Worker selection ────────────────────────────────────────────────────────── + +async function selectEnrichmentWorker(id) { + enrichmentManagerState.selected = id; + enrichmentManagerState.breakdown = null; + enrichmentManagerState.unmatched = null; + enrichmentManagerState.search = ''; + enrichmentManagerState.page = 0; + enrichmentManagerState.statusFilter = 'unmatched'; + _emHighlightRail(); + + // Pick a default entity tab the worker actually supports (filled after the + // unmatched call returns entity_types; default to artist meanwhile). + enrichmentManagerState.entityTab = 'artist'; + renderEnrichmentPanel(); + await Promise.all([_emLoadBreakdown(id), _emLoadUnmatched()]); + renderEnrichmentPanel(); +} + +async function _emLoadBreakdown(id) { + try { + const res = await fetch(`/api/enrichment/${id}/breakdown`); + enrichmentManagerState.breakdown = res.ok ? (await res.json()).breakdown : null; + } catch (_e) { + enrichmentManagerState.breakdown = null; + } +} + +async function _emLoadUnmatched() { + const id = enrichmentManagerState.selected; + const token = ++enrichmentManagerState.loadToken; + const { entityTab, statusFilter, search, page, pageSize } = enrichmentManagerState; + const params = new URLSearchParams({ + entity_type: entityTab, + status: statusFilter, + limit: String(pageSize), + offset: String(page * pageSize), + }); + if (search) params.set('q', search); + try { + const res = await fetch(`/api/enrichment/${id}/unmatched?${params}`); + const data = res.ok ? await res.json() : { total: 0, items: [] }; + if (token !== enrichmentManagerState.loadToken) return; // stale + enrichmentManagerState.unmatched = data; + } catch (_e) { + if (token === enrichmentManagerState.loadToken) { + enrichmentManagerState.unmatched = { total: 0, items: [] }; + } + } +} + +// ── Detail panel ────────────────────────────────────────────────────────────── + +function renderEnrichmentPanel() { + const panel = document.getElementById('em-panel'); + if (!panel) return; + const id = enrichmentManagerState.selected; + const worker = _emWorkerById[id]; + if (!worker) { panel.innerHTML = ''; return; } + + // Theme the whole panel to the selected worker's accent colour. + panel.style.setProperty('--em-accent', worker.color); + panel.style.setProperty('--em-accent-rgb', _emHexToRgb(worker.color)); + + panel.innerHTML = ` +
+
Enrichment coverage
+
+
+
+
+
+
`; + _emRenderPanelHeader(); + _emRenderStats(); + _emRenderUnmatchedControls(); + _emRenderUnmatchedList(); +} + +function _emRenderPanelHeader() { + const host = document.getElementById('em-panel-header'); + if (!host) return; + const id = enrichmentManagerState.selected; + const worker = _emWorkerById[id]; + // Structure is rendered once per worker selection; the live bits below + // (pill / current-item / errors / toggle) are updated in place by + // _emUpdateHeaderLive on each poll so the logo never reflows or flickers. + host.innerHTML = ` +
+
+ ${_emIconHtml(id, 'lg')} +
+
${_emEscape(worker.name)} enrichment
+
+
+
+
+ + + + +
+
`; + _emUpdateHeaderLive(); +} + +function _emUpdateHeaderLive() { + const id = enrichmentManagerState.selected; + const status = enrichmentManagerState.statuses[id]; + const info = _emStatusInfo(status); + + const pill = document.getElementById('em-ph-pill'); + if (pill) { pill.className = `em-pill em-pill--${info.cls}`; pill.textContent = info.label; } + + const metric = document.getElementById('em-ph-metric'); + if (metric) { + const pct = _emOverallPct(status); + metric.innerHTML = pct == null + ? '' + : `${pct}% + enriched`; + } + + const cur = document.getElementById('em-ph-current'); + if (cur) { + const item = status && status.current_item; + cur.innerHTML = item + ? `Now enriching: ${_emEscape(item.name || '')}${item.type ? ` (${_emEscape(item.type)})` : ''}` + : 'No item processing'; + } + + const budgetEl = document.getElementById('em-ph-budget'); + if (budgetEl) { + const b = status && status.daily_budget; + budgetEl.innerHTML = (b && b.limit) + ? `Budget ${b.used ?? '?'} / ${b.limit}` : ''; + } + + const errEl = document.getElementById('em-ph-errors'); + if (errEl) { + const errors = (status && status.stats && status.stats.errors) || 0; + errEl.innerHTML = errors ? `⚠ ${errors}` : ''; + } + + const toggle = document.getElementById('em-ph-toggle'); + if (toggle) { + const isPaused = status && status.paused; + toggle.disabled = !(status && status.enabled); + toggle.classList.toggle('em-btn--go', !!isPaused); + toggle.textContent = isPaused ? '▶ Resume' : '⏸ Pause'; + } +} + +function _emRenderStats() { + const host = document.getElementById('em-stats'); + if (!host) return; + const bd = enrichmentManagerState.breakdown; + if (!bd) { + // Skeleton cards (count unknown yet — 3 covers the common case). + host.innerHTML = Array.from({ length: 3 }, () => ` +
+
+
+
+
`).join(''); + return; + } + + const glyphs = { artist: '🎤', album: '💿', track: '🎵' }; + host.innerHTML = Object.keys(bd).map(entity => { + const d = bd[entity] || {}; + const total = d.total || 0; + const matched = d.matched || 0; + const notFound = d.not_found || 0; + const pending = d.pending || 0; + const pct = total ? Math.round((matched / total) * 100) : 0; + const seg = (n) => (total ? (n / total) * 100 : 0); + return ` +
+
+ ${glyphs[entity] || '•'}${_emEntityLabel(entity, true)} + ${pct}% +
+
+
+
+
+
+
+ ${matched.toLocaleString()} matched + ${notFound.toLocaleString()} missed + ${pending.toLocaleString()} pending +
+
`; + }).join(''); + + // Animate the segments in from 0 on the next frame (CSS transition does the rest). + requestAnimationFrame(() => { + host.querySelectorAll('.em-seg-fill').forEach(el => { + el.style.width = `${el.dataset.pct || 0}%`; + }); + }); +} + +function _emRenderUnmatchedControls() { + const host = document.getElementById('em-unmatched-controls'); + if (!host) return; + const data = enrichmentManagerState.unmatched; + const supported = (data && data.entity_types) || ['artist']; + const total = data ? (data.total || 0) : null; + const tabs = supported.map(e => ` + `).join(''); + + host.innerHTML = ` +
+
+ Needs matching + ${total == null ? '' : `${total.toLocaleString()}`} +
+
+
${tabs}
+ +
+ ⌕ + +
+
+
`; +} + +function _emRenderUnmatchedList() { + const host = document.getElementById('em-unmatched-list'); + if (!host) return; + const data = enrichmentManagerState.unmatched; + if (!data) { + host.innerHTML = Array.from({ length: 6 }, () => ` +
+
+
+
+
+
+
`).join(''); + return; + } + // Keep the count badge in sync without re-rendering the controls (would + // steal focus from the search box mid-type). + const countEl = document.querySelector('#em-unmatched-controls .em-count'); + if (countEl) countEl.textContent = (data.total || 0).toLocaleString(); + + if (!data.items.length) { + const allMatched = enrichmentManagerState.statusFilter === 'unmatched'; + host.innerHTML = `
+
${allMatched ? '🎉' : '🔍'}
+
${allMatched + ? 'Every item is matched for this source.' + : 'Nothing matches this filter.'}
+
`; + } else { + const id = enrichmentManagerState.selected; + const entity = enrichmentManagerState.entityTab; + host.innerHTML = data.items.map(item => { + const img = item.image_url + ? `` + : '
♪
'; + const rel = _emRelativeTime(item.last_attempted); + const last = rel + ? `tried ${rel}` + : 'never tried'; + const statusBadge = item.status === 'not_found' + ? 'not found' + : 'pending'; + const safeName = _emEscape(item.name || 'Unknown'); + return ` +
+ ${img} +
+
${safeName}
+
${statusBadge} ${last}
+
+
+ + +
+
`; + }).join(''); + } + _emRenderPager(); +} + +function _emRenderPager() { + const host = document.getElementById('em-pager'); + if (!host) return; + const data = enrichmentManagerState.unmatched; + if (!data) { host.innerHTML = ''; return; } + const { page, pageSize } = enrichmentManagerState; + const total = data.total || 0; + const from = total ? page * pageSize + 1 : 0; + const to = Math.min((page + 1) * pageSize, total); + const hasPrev = page > 0; + const hasNext = to < total; + host.innerHTML = ` + + ${from}–${to} of ${total.toLocaleString()} + `; +} + +// ── Controls ────────────────────────────────────────────────────────────────── + +async function setEnrichmentEntityTab(entity) { + enrichmentManagerState.entityTab = entity; + enrichmentManagerState.page = 0; + _emRenderUnmatchedControls(); + document.getElementById('em-unmatched-list').innerHTML = '
'; + await _emLoadUnmatched(); + _emRenderUnmatchedList(); +} + +async function setEnrichmentStatusFilter(value) { + enrichmentManagerState.statusFilter = value; + enrichmentManagerState.page = 0; + await _emLoadUnmatched(); + _emRenderUnmatchedList(); +} + +let _emSearchDebounce = null; +function onEnrichmentSearchInput(value) { + enrichmentManagerState.search = value; + enrichmentManagerState.page = 0; + if (_emSearchDebounce) clearTimeout(_emSearchDebounce); + _emSearchDebounce = setTimeout(async () => { + await _emLoadUnmatched(); + _emRenderUnmatchedList(); + }, 300); +} + +async function changeEnrichmentPage(delta) { + enrichmentManagerState.page = Math.max(0, enrichmentManagerState.page + delta); + await _emLoadUnmatched(); + _emRenderUnmatchedList(); +} + +async function toggleEnrichmentWorker(id) { + const status = enrichmentManagerState.statuses[id]; + const action = status?.paused ? 'resume' : 'pause'; + try { + const res = await fetch(`/api/enrichment/${id}/${action}`, { method: 'POST' }); + const data = await res.json().catch(() => ({})); + if (!res.ok) { + showToast(data.error || `Could not ${action} worker`, 'error'); + return; + } + showToast(`${_emWorkerById[id].name} ${action === 'pause' ? 'paused' : 'resumed'}`, 'success'); + await _emPollSelected(); + } catch (_e) { + showToast(`Error trying to ${action} worker`, 'error'); + } +} + +async function retryEnrichmentItem(service, entityType, entityId, btn) { + if (btn) { btn.disabled = true; btn.textContent = '…'; } + try { + const res = await fetch('/api/library/clear-match', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ entity_type: entityType, entity_id: entityId, service }), + }); + const data = await res.json().catch(() => ({})); + if (data.success) { + showToast('Re-queued for enrichment', 'success'); + await Promise.all([_emLoadBreakdown(service), _emLoadUnmatched()]); + _emRenderStats(); + _emRenderUnmatchedList(); + } else { + showToast(data.error || 'Failed to re-queue', 'error'); + if (btn) { btn.disabled = false; btn.textContent = 'Retry'; } + } + } catch (_e) { + showToast('Error re-queuing item', 'error'); + if (btn) { btn.disabled = false; btn.textContent = 'Retry'; } + } +} + +// ── Inline manual match (decoupled from the library artist-detail page) ─────── + +function openEnrichmentMatch(service, entityType, entityId, anchorBtn) { + const defaultQuery = anchorBtn + ? (anchorBtn.closest('.em-row')?.querySelector('.em-row-name')?.textContent || '') + : ''; + const existing = document.getElementById('enrichment-match-overlay'); + if (existing) existing.remove(); + + const overlay = document.createElement('div'); + overlay.id = 'enrichment-match-overlay'; + overlay.className = 'modal-overlay'; + overlay.style.zIndex = '10010'; // above the manager modal + overlay.onclick = (e) => { if (e.target === overlay) overlay.remove(); }; + overlay.innerHTML = ` +
+
+

Match ${_emEntityLabel(entityType)} on ${_emEscape(_emWorkerById[service]?.name || service)}

+ +
+
+ + +
+
+
Search to find a match.
+
+
`; + document.body.appendChild(overlay); + + const input = overlay.querySelector('.enhanced-match-search-input'); + const results = overlay.querySelector('#enrichment-match-results'); + overlay.querySelector('.enhanced-bulk-modal-close').onclick = () => overlay.remove(); + const run = () => _emRunMatchSearch(service, entityType, entityId, input.value, results, overlay); + overlay.querySelector('.em-match-go').onclick = run; + input.addEventListener('keydown', (e) => { if (e.key === 'Enter') run(); }); + if (defaultQuery.trim()) run(); + setTimeout(() => input.focus(), 50); +} + +async function _emRunMatchSearch(service, entityType, entityId, query, container, overlay) { + if (!query.trim()) { + container.innerHTML = '
Enter a search term
'; + return; + } + container.innerHTML = '
'; + try { + const res = await fetch('/api/library/search-service', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ service, entity_type: entityType, query: query.trim() }), + }); + const data = await res.json(); + if (!data.success) throw new Error(data.error || 'Search failed'); + const list = data.results || []; + if (!list.length) { + container.innerHTML = '
No results. Try a different search.
'; + return; + } + container.innerHTML = ''; + list.forEach(r => { + const row = document.createElement('div'); + row.className = 'enhanced-match-result-row'; + const imgHtml = r.image + ? `` + : '
🎵
'; + const providerLabel = r.provider && r.provider !== service ? ` (${_emEscape(r.provider)})` : ''; + row.innerHTML = ` + ${imgHtml} +
+
${_emEscape(r.name || 'Unknown')}
+ ${r.extra ? `
${_emEscape(r.extra)}
` : ''} +
ID: ${_emEscape(r.id)}${providerLabel}
+
`; + const btn = document.createElement('button'); + btn.className = 'enhanced-meta-save-btn'; + btn.textContent = 'Match'; + btn.onclick = () => _emApplyMatch(entityType, entityId, r.provider || service, r.id, overlay); + row.appendChild(btn); + container.appendChild(row); + }); + } catch (e) { + container.innerHTML = `
Search error: ${_emEscape(e.message)}
`; + } +} + +async function _emApplyMatch(entityType, entityId, service, serviceId, overlay) { + try { + const res = await fetch('/api/library/manual-match', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ entity_type: entityType, entity_id: entityId, service, service_id: serviceId }), + }); + const data = await res.json(); + if (data.success) { + showToast('Matched ✓', 'success'); + if (overlay) overlay.remove(); + // Refresh the manager's stats + list for the *selected* worker. + const sel = enrichmentManagerState.selected; + await Promise.all([_emLoadBreakdown(sel), _emLoadUnmatched()]); + _emRenderStats(); + _emRenderUnmatchedList(); + } else { + showToast(data.error || 'Failed to match', 'error'); + } + } catch (_e) { + showToast('Error applying match', 'error'); + } +} + +// Expose for inline onclick handlers. +window.openEnrichmentManager = openEnrichmentManager; +window.closeEnrichmentManager = closeEnrichmentManager; +window.refreshEnrichmentManager = refreshEnrichmentManager; +window.selectEnrichmentWorker = selectEnrichmentWorker; +window.setEnrichmentEntityTab = setEnrichmentEntityTab; +window.setEnrichmentStatusFilter = setEnrichmentStatusFilter; +window.onEnrichmentSearchInput = onEnrichmentSearchInput; +window.changeEnrichmentPage = changeEnrichmentPage; +window.toggleEnrichmentWorker = toggleEnrichmentWorker; +window.retryEnrichmentItem = retryEnrichmentItem; +window.openEnrichmentMatch = openEnrichmentMatch; diff --git a/webui/static/style.css b/webui/static/style.css index 2324a5c3..0fa81c2a 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -64744,3 +64744,388 @@ body.reduce-effects .dash-card::after { background: rgba(var(--accent-rgb), 0.3); background-clip: padding-box; } + +/* =========================================================================== + Manage Enrichment Workers modal (enrichment-manager.js) + =========================================================================== */ +.em-manage-btn { + display: inline-flex; + align-items: center; + gap: 9px; + height: 44px; + padding: 0 18px 0 8px; + margin-left: 10px; + background: linear-gradient(135deg, rgba(var(--accent-rgb), 0.16) 0%, rgba(var(--accent-rgb), 0.07) 100%); + backdrop-filter: blur(20px) saturate(1.4); + -webkit-backdrop-filter: blur(20px) saturate(1.4); + border: 1.5px solid rgba(var(--accent-rgb), 0.28); + border-radius: 999px; + color: #fff; + font-size: 13.5px; + font-weight: 700; + letter-spacing: 0.2px; + cursor: pointer; + transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1); + white-space: nowrap; + box-shadow: + 0 4px 16px rgba(var(--accent-rgb), 0.18), + 0 2px 8px rgba(0,0,0,0.15), + inset 0 1px 0 rgba(255,255,255,0.08); +} +.em-manage-btn:hover { + border-color: rgba(var(--accent-rgb), 0.5); + transform: scale(1.04); + box-shadow: + 0 6px 22px rgba(var(--accent-rgb), 0.32), + 0 3px 12px rgba(0,0,0,0.2), + inset 0 1px 0 rgba(255,255,255,0.12); +} +.em-manage-btn-icon { + display: flex; + align-items: center; + justify-content: center; + width: 30px; + height: 30px; + border-radius: 50%; + font-size: 16px; + background: linear-gradient(135deg, rgba(var(--accent-rgb), 0.9), rgba(var(--accent-rgb), 0.55)); + box-shadow: 0 0 12px rgba(var(--accent-rgb), 0.5); +} +.em-manage-btn-label { background: linear-gradient(90deg, #fff, rgba(255,255,255,0.85)); -webkit-background-clip: text; background-clip: text; } + +.enrichment-manager-modal { + position: relative; + background: + radial-gradient(120% 80% at 0% 0%, rgba(var(--accent-rgb), 0.10), transparent 55%), + radial-gradient(100% 70% at 100% 0%, rgba(255,255,255,0.04), transparent 50%), + linear-gradient(150deg, #1c1c1f 0%, #131316 55%, #0f0f12 100%); + border-radius: 18px; + border: 1px solid rgba(255,255,255,0.09); + width: 1150px; + max-width: 95vw; + height: 82vh; + max-height: 860px; + display: flex; + flex-direction: column; + box-shadow: + 0 30px 90px rgba(0,0,0,0.65), + 0 0 0 1px rgba(var(--accent-rgb), 0.12), + inset 0 1px 0 rgba(255,255,255,0.06); + overflow: hidden; +} +/* Hairline top accent line across the whole modal. */ +.enrichment-manager-modal::before { + content: ''; position: absolute; top: 0; left: 0; right: 0; height: 1px; + background: linear-gradient(90deg, transparent, rgba(var(--accent-rgb), 0.6), transparent); + z-index: 2; +} +.enrichment-manager-modal .enhanced-bulk-modal-header { + flex: 0 0 auto; + background: linear-gradient(135deg, rgba(var(--accent-rgb), 0.14), transparent 70%); +} +.enrichment-manager-modal .enhanced-bulk-modal-header h3 { letter-spacing: 0.3px; } +.em-body { display: flex; flex: 1 1 auto; min-height: 0; } + +/* Left rail */ +.em-rail { + flex: 0 0 230px; + border-right: 1px solid rgba(255,255,255,0.07); + padding: 12px; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 4px; +} +.em-worker-row { + display: flex; + align-items: center; + gap: 10px; + padding: 9px 10px; + background: transparent; + border: 1px solid transparent; + border-radius: 10px; + cursor: pointer; + text-align: left; + transition: background 0.15s ease, border-color 0.15s ease; +} +.em-worker-row { position: relative; } +.em-worker-row:hover { background: rgba(255,255,255,0.05); } +.em-worker-row.active { + background: rgba(var(--accent-rgb), 0.14); + border-color: rgba(var(--accent-rgb), 0.4); +} +.em-worker-row.active::before { + content: ''; + position: absolute; left: -1px; top: 8px; bottom: 8px; width: 3px; + border-radius: 0 3px 3px 0; + background: rgb(var(--accent-rgb)); + box-shadow: 0 0 10px rgba(var(--accent-rgb), 0.7); +} +/* Circular glowing logo chip — mirrors the dashboard enrichment bubbles. */ +.em-icon { + flex: 0 0 auto; + width: 34px; height: 34px; + border-radius: 50%; + display: flex; align-items: center; justify-content: center; + background: linear-gradient(135deg, rgba(255,255,255,0.07), rgba(255,255,255,0.02)); + border: 1.5px solid color-mix(in srgb, var(--em-accent, #888) 45%, transparent); + box-shadow: 0 0 12px color-mix(in srgb, var(--em-accent, #888) 30%, transparent), + inset 0 1px 0 rgba(255,255,255,0.08); + overflow: hidden; + transition: transform 0.25s cubic-bezier(0.4, 0, 0.2, 1); +} +.em-icon--lg { width: 52px; height: 52px; border-width: 2px; } +.em-icon-img { width: 66%; height: 66%; object-fit: contain; display: block; } +.em-icon-letter { font-weight: 800; font-size: 15px; color: var(--em-accent, #fff); text-shadow: 0 1px 2px rgba(0,0,0,0.5); } +.em-icon--lg .em-icon-letter { font-size: 22px; } +.em-worker-row:hover .em-icon { transform: scale(1.08); } +.em-worker-meta { flex: 1 1 auto; min-width: 0; display: flex; flex-direction: column; gap: 2px; } +.em-worker-name { color: #eee; font-size: 13px; font-weight: 600; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +.em-worker-sub { font-size: 10.5px; color: rgba(255,255,255,0.4); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +.em-rail-cov { display: block; height: 3px; border-radius: 999px; background: rgba(255,255,255,0.1); overflow: hidden; margin-top: 2px; } +.em-rail-cov-fill { display: block; height: 100%; border-radius: 999px; background: color-mix(in srgb, var(--accent, #4ade80) 70%, transparent); background: rgb(var(--accent-rgb)); transition: width 0.5s cubic-bezier(0.4,0,0.2,1); } +.em-dot { flex: 0 0 auto; width: 9px; height: 9px; border-radius: 50%; background: #555; } +.em-dot--running { background: #1db954; box-shadow: 0 0 8px #1db954; } +.em-dot--idle { background: #4a90d9; } +.em-dot--paused { background: #e0a93b; } +.em-dot--ratelimited { background: #e05b5b; box-shadow: 0 0 8px #e05b5b; } +.em-dot--disabled, .em-dot--stopped { background: #555; } + +/* Right panel */ +.em-panel { + flex: 1 1 auto; + min-width: 0; + padding: 18px 22px; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 16px; +} +.em-panel-header { flex: 0 0 auto; } +.em-ph-top { display: flex; align-items: center; gap: 14px; } +.em-ph-titles { flex: 1 1 auto; min-width: 0; } +.em-ph-name { font-size: 19px; font-weight: 800; color: #fff; } +.em-ph-sub { font-size: 13px; color: rgba(255,255,255,0.7); margin-top: 2px; } +.em-ph-actions { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; justify-content: flex-end; } +.em-muted { color: rgba(255,255,255,0.45); } + +.em-pill { + padding: 4px 11px; border-radius: 999px; font-size: 11px; font-weight: 700; + text-transform: uppercase; letter-spacing: 0.4px; +} +.em-pill--running { background: rgba(29,185,84,0.18); color: #4ade80; } +.em-pill--idle { background: rgba(74,144,217,0.18); color: #7fb5ec; } +.em-pill--paused { background: rgba(224,169,59,0.18); color: #f0c060; } +.em-pill--ratelimited { background: rgba(224,91,91,0.2); color: #ff8b8b; } +.em-pill--disabled, .em-pill--stopped { background: rgba(255,255,255,0.08); color: rgba(255,255,255,0.5); } + +.em-chip { + padding: 4px 9px; border-radius: 8px; font-size: 11px; font-weight: 600; + background: rgba(255,255,255,0.08); color: rgba(255,255,255,0.75); +} +.em-chip--err { background: rgba(224,91,91,0.18); color: #ff8b8b; } +.em-chip--nf { background: rgba(224,91,91,0.15); color: #ff9b9b; } +.em-chip--pend { background: rgba(224,169,59,0.15); color: #f0c060; } + +.em-btn { + padding: 8px 14px; border-radius: 9px; border: 1px solid rgba(var(--accent-rgb), 0.4); + background: rgba(var(--accent-rgb), 0.15); color: #fff; font-size: 13px; font-weight: 600; + cursor: pointer; transition: all 0.15s ease; +} +.em-btn:hover:not(:disabled) { background: rgba(var(--accent-rgb), 0.28); } +.em-btn:disabled { opacity: 0.4; cursor: not-allowed; } +.em-btn--go { background: rgba(29,185,84,0.2); border-color: rgba(29,185,84,0.5); } +.em-btn--sm { padding: 5px 10px; font-size: 12px; } +.em-btn--ghost { background: transparent; border-color: rgba(255,255,255,0.18); color: rgba(255,255,255,0.7); } +.em-btn--ghost:hover:not(:disabled) { background: rgba(255,255,255,0.08); } + +/* Stat cards */ +.em-stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(190px, 1fr)); gap: 12px; flex: 0 0 auto; } +.em-stat-card { + background: linear-gradient(160deg, rgba(255,255,255,0.06), rgba(255,255,255,0.02)); + border: 1px solid rgba(255,255,255,0.08); + border-radius: 12px; padding: 14px; + transition: transform 0.2s ease, border-color 0.2s ease; +} +.em-stat-card:hover { transform: translateY(-2px); border-color: rgba(var(--accent-rgb), 0.3); } +.em-bar-fill { box-shadow: 0 0 10px rgba(var(--accent-rgb), 0.5); } +.em-stat-head { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 8px; } +.em-stat-title { font-size: 12px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.5px; color: rgba(255,255,255,0.6); } +.em-stat-pct { font-size: 20px; font-weight: 800; color: #fff; } +.em-bar { height: 7px; border-radius: 999px; background: rgba(255,255,255,0.1); overflow: hidden; } +.em-bar-fill { height: 100%; border-radius: 999px; background: linear-gradient(90deg, rgba(var(--accent-rgb),0.7), rgba(var(--accent-rgb),1)); transition: width 0.4s ease; } +.em-stat-legend { display: flex; gap: 10px; margin-top: 9px; font-size: 11px; flex-wrap: wrap; } +.em-leg--matched { color: #4ade80; } +.em-leg--nf { color: #ff9b9b; } +.em-leg--pend { color: #f0c060; } + +/* Unmatched browser */ +.em-unmatched { flex: 1 1 auto; display: flex; flex-direction: column; min-height: 0; gap: 10px; } +.em-unmatched-controls { display: flex; justify-content: space-between; align-items: center; gap: 12px; flex-wrap: wrap; flex: 0 0 auto; } +.em-tabs { display: flex; gap: 6px; } +.em-tab { + padding: 7px 14px; border-radius: 8px; border: 1px solid rgba(255,255,255,0.1); + background: transparent; color: rgba(255,255,255,0.65); font-size: 13px; font-weight: 600; cursor: pointer; +} +.em-tab.active { background: rgba(var(--accent-rgb), 0.16); border-color: rgba(var(--accent-rgb), 0.4); color: #fff; } +.em-filter-row { display: flex; gap: 8px; align-items: center; } +.em-select, .em-search { + padding: 8px 12px; background: rgba(255,255,255,0.06); border: 1px solid rgba(255,255,255,0.12); + border-radius: 8px; color: #fff; font-size: 13px; +} +.em-search { min-width: 200px; } +.em-search:focus, .em-select:focus { outline: none; border-color: rgba(var(--accent-rgb), 0.6); } + +.em-unmatched-list { flex: 1 1 auto; overflow-y: auto; display: flex; flex-direction: column; gap: 6px; padding-right: 4px; } +.em-row { + display: flex; align-items: center; gap: 12px; padding: 9px 12px; + background: rgba(255,255,255,0.03); border: 1px solid rgba(255,255,255,0.06); border-radius: 10px; +} +.em-row:hover { background: rgba(255,255,255,0.06); } +.em-row-img { width: 42px; height: 42px; border-radius: 8px; object-fit: cover; flex: 0 0 auto; } +.em-row-img--ph { display: flex; align-items: center; justify-content: center; background: rgba(255,255,255,0.08); color: rgba(255,255,255,0.4); font-size: 18px; } +.em-row-info { flex: 1 1 auto; min-width: 0; } +.em-row-name { font-size: 14px; font-weight: 600; color: #fff; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +.em-row-meta { display: flex; gap: 8px; align-items: center; margin-top: 3px; font-size: 11px; } +.em-row-actions { display: flex; gap: 6px; flex: 0 0 auto; } +.em-empty { text-align: center; padding: 48px 20px; color: rgba(255,255,255,0.55); font-size: 15px; } +.em-empty-emoji { font-size: 38px; margin-bottom: 10px; opacity: 0.85; } +.em-pager { display: flex; align-items: center; justify-content: center; gap: 14px; flex: 0 0 auto; padding-top: 4px; font-size: 12px; } + +/* --- Motion: entrance / exit + scroll lock --- */ +body.em-scroll-lock { overflow: hidden; } +.em-overlay { transition: opacity 0.22s ease, backdrop-filter 0.22s ease; } +.em-overlay.em-closing { opacity: 0; } +@keyframes em-pop-in { + from { opacity: 0; transform: translateY(14px) scale(0.97); } + to { opacity: 1; transform: translateY(0) scale(1); } +} +.enrichment-manager-modal.em-in { animation: em-pop-in 0.28s cubic-bezier(0.16, 1, 0.3, 1) both; } +.em-overlay.em-closing .enrichment-manager-modal { transform: scale(0.98); opacity: 0; transition: all 0.16s ease; } +.enrichment-manager-modal:focus { outline: none; } + +/* --- Header actions / refresh --- */ +.em-header-actions { display: flex; align-items: center; gap: 8px; } +.em-icon-btn { + width: 32px; height: 32px; border-radius: 50%; + background: rgba(255,255,255,0.08); color: rgba(255,255,255,0.8); + border: none; cursor: pointer; font-size: 16px; line-height: 1; + display: flex; align-items: center; justify-content: center; + transition: background 0.2s ease, transform 0.2s ease; +} +.em-icon-btn:hover { background: rgba(255,255,255,0.16); transform: rotate(15deg); } +.em-icon-btn.em-spinning { animation: em-spin 0.6s linear; } +@keyframes em-spin { to { transform: rotate(360deg); } } + +.em-stat-pct-sym { font-size: 13px; opacity: 0.6; margin-left: 1px; } + +/* --- Skeleton loaders --- */ +.em-skel { + position: relative; overflow: hidden; + background: rgba(255,255,255,0.06); border-radius: 6px; +} +.em-skel::after { + content: ''; position: absolute; inset: 0; + background: linear-gradient(90deg, transparent, rgba(255,255,255,0.10), transparent); + transform: translateX(-100%); animation: em-shimmer 1.3s infinite; +} +@keyframes em-shimmer { 100% { transform: translateX(100%); } } +.em-skel-line { height: 11px; } +.em-skel-bar { height: 7px; margin: 12px 0; border-radius: 999px; } +.em-skel-card { display: flex; flex-direction: column; } +.em-skel-row .em-row-img { background: rgba(255,255,255,0.06); } +.em-skel-row { pointer-events: none; } + +@media (prefers-reduced-motion: reduce) { + .enrichment-manager-modal.em-in, + .em-icon-btn.em-spinning, + .em-skel::after { animation: none; } + .em-bar-fill, .em-rail-cov-fill { transition: none; } +} + +/* --- Narrow screens: rail becomes a horizontal strip --- */ +@media (max-width: 760px) { + .enrichment-manager-modal { height: 90vh; } + .em-body { flex-direction: column; } + .em-rail { flex: 0 0 auto; flex-direction: row; overflow-x: auto; border-right: none; border-bottom: 1px solid rgba(255,255,255,0.07); } + .em-worker-row { flex: 0 0 auto; } + .em-worker-meta { display: none; } +} + +/* ===== Panel polish: hero header, accent theming, segmented stats ===== */ +/* The panel sets --em-accent / --em-accent-rgb to the selected worker colour. */ +.em-section-label { + font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.8px; + color: rgba(255,255,255,0.4); flex: 0 0 auto; +} +.em-section-label--inline { display: flex; align-items: center; gap: 8px; } +.em-count { + display: inline-flex; align-items: center; justify-content: center; + min-width: 22px; height: 19px; padding: 0 7px; border-radius: 999px; + background: rgba(var(--em-accent-rgb, 99,102,241), 0.2); + color: rgb(var(--em-accent-rgb, 129,140,248)); + font-size: 11px; font-weight: 800; letter-spacing: 0; +} + +/* Hero header */ +.em-hero { + position: relative; overflow: hidden; + display: flex; align-items: center; gap: 16px; + padding: 18px 20px; + border-radius: 16px; + background: + linear-gradient(135deg, rgba(var(--em-accent-rgb, 99,102,241), 0.16), rgba(var(--em-accent-rgb, 99,102,241), 0.03) 60%), + rgba(255,255,255,0.03); + border: 1px solid rgba(var(--em-accent-rgb, 99,102,241), 0.22); +} +.em-hero-glow { + position: absolute; top: -60%; right: -10%; width: 300px; height: 300px; + background: radial-gradient(circle, rgba(var(--em-accent-rgb, 99,102,241), 0.22), transparent 70%); + pointer-events: none; +} +.em-hero .em-icon { width: 56px; height: 56px; } +.em-hero .em-ph-titles { flex: 1 1 auto; min-width: 0; z-index: 1; } +.em-ph-name-sub { font-size: 14px; font-weight: 500; color: rgba(255,255,255,0.45); } +.em-hero-metric { display: flex; flex-direction: column; align-items: center; justify-content: center; line-height: 1; z-index: 1; padding: 0 6px; } +.em-hero-pct { font-size: 30px; font-weight: 800; color: #fff; } +.em-hero-pct-sym { font-size: 16px; opacity: 0.6; } +.em-hero-pct-label { font-size: 10px; text-transform: uppercase; letter-spacing: 0.6px; color: rgba(255,255,255,0.45); margin-top: 3px; } + +/* Accent-themed buttons / active states inside the panel */ +.em-panel .em-btn { border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.45); background: rgba(var(--em-accent-rgb, 99,102,241), 0.16); } +.em-panel .em-btn:hover:not(:disabled) { background: rgba(var(--em-accent-rgb, 99,102,241), 0.3); } + +/* Entity glyph in stat-card titles */ +.em-stat-title { display: inline-flex; align-items: center; gap: 7px; } +.em-stat-ico { font-size: 14px; filter: saturate(0.9); } + +/* Segmented matched/not-found/pending bar */ +.em-seg { display: flex; height: 9px; border-radius: 999px; overflow: hidden; background: rgba(255,255,255,0.07); } +.em-seg-fill { height: 100%; transition: width 0.6s cubic-bezier(0.16,1,0.3,1); } +.em-seg--matched { background: linear-gradient(90deg, rgba(var(--em-accent-rgb, 74,222,128),0.85), rgb(var(--em-accent-rgb, 74,222,128))); } +.em-seg--nf { background: #e0586b; } +.em-seg--pend { background: rgba(240,192,96,0.85); } +.em-stat-legend .em-leg { display: inline-flex; align-items: center; gap: 5px; } +.em-stat-legend .em-leg i { width: 8px; height: 8px; border-radius: 50%; display: inline-block; } +.em-leg--matched i { background: rgb(var(--em-accent-rgb, 74,222,128)); } +.em-leg--nf i { background: #e0586b; } +.em-leg--pend i { background: rgba(240,192,96,0.9); } + +/* Unmatched toolbar */ +.em-unmatched-bar { display: flex; align-items: center; justify-content: space-between; gap: 14px; flex-wrap: wrap; flex: 0 0 auto; } +.em-seg-tabs { display: inline-flex; padding: 3px; gap: 2px; background: rgba(255,255,255,0.05); border-radius: 10px; border: 1px solid rgba(255,255,255,0.07); } +.em-seg-tab { + padding: 6px 14px; border-radius: 8px; border: none; background: transparent; + color: rgba(255,255,255,0.6); font-size: 12.5px; font-weight: 600; cursor: pointer; + transition: all 0.18s ease; +} +.em-seg-tab:hover { color: #fff; } +.em-seg-tab.active { + background: rgba(var(--em-accent-rgb, 99,102,241), 0.9); + color: #fff; box-shadow: 0 2px 8px rgba(var(--em-accent-rgb, 99,102,241), 0.4); +} +.em-search-wrap { position: relative; display: inline-flex; align-items: center; } +.em-search-ico { position: absolute; left: 11px; color: rgba(255,255,255,0.4); font-size: 15px; pointer-events: none; } +.em-search-wrap .em-search { padding-left: 30px; } +.em-search:focus, .em-select:focus { outline: none; border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.6); } +.em-row:hover { background: rgba(var(--em-accent-rgb, 255,255,255), 0.07); border-color: rgba(var(--em-accent-rgb, 255,255,255), 0.14); } From fc9a9f1c90b1a4b3a39bf71dc253b07aa6f60f85 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 19:19:39 -0700 Subject: [PATCH 024/108] Enrichment manager v2: working retry + bulk retry-all-failed Fixes a correctness bug and adds bulk re-queuing. - Bug: per-row 'Retry' used clear-match, which sets an item to not_found with last_attempted=NULL. The worker only retries not_found items where last_attempted < (now - 30d), and 'NULL < cutoff' is false in SQLite, so those items were never re-queued. Fixed by resetting match_status to NULL (pending), which every worker's queue picks up on the next pass. - New POST /api/enrichment//retry with scope 'item' | 'failed' (failed = re-queue every not_found item of an entity type), backed by a pure whitelisted build_reset_query + MusicDatabase.reset_enrichment(). - UI: per-row Retry now hits /retry; a 'Retry all failed' bulk button appears when the current entity has not-found items (confirm + count toast); a hint line explains retry/match/auto-retry behaviour. - 11 new tests (38 enrichment tests total, all green). --- core/enrichment/api.py | 27 ++++++++++++++ core/enrichment/unmatched.py | 42 +++++++++++++++++++++ database/music_database.py | 18 +++++++++ tests/enrichment/test_unmatched.py | 59 ++++++++++++++++++++++++++++++ webui/static/enrichment-manager.js | 54 ++++++++++++++++++++++++--- webui/static/style.css | 5 +++ 6 files changed, 200 insertions(+), 5 deletions(-) diff --git a/core/enrichment/api.py b/core/enrichment/api.py index 7cf472f7..746e0bd5 100644 --- a/core/enrichment/api.py +++ b/core/enrichment/api.py @@ -223,4 +223,31 @@ def create_blueprint() -> Blueprint: }) return jsonify(result), 200 + @bp.route('/api/enrichment//retry', methods=['POST']) + def enrichment_retry(service_id: str): + """Re-queue item(s) so the worker re-attempts them. + + Body: ``entity_type`` (artist|album|track), ``scope`` (item|failed), + ``entity_id`` (required when scope='item'). 'failed' re-queues every + not_found item of that entity type. + """ + if service_id not in SERVICE_ENTITY_SUPPORT: + return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404 + if _db_getter is None: + return jsonify({'error': 'database unavailable'}), 503 + + data = request.get_json(silent=True) or {} + entity_type = (data.get('entity_type') or 'artist').strip() + scope = (data.get('scope') or 'item').strip() + entity_id = data.get('entity_id') + try: + count = _db_getter().reset_enrichment(service_id, entity_type, scope, entity_id) + except UnmatchedQueryError as e: + return jsonify({'error': str(e)}), 400 + except Exception as e: + logger.error("Error re-queuing %s %s (%s): %s", service_id, entity_type, scope, e) + return jsonify({'error': str(e)}), 500 + return jsonify({'success': True, 'reset': count, 'service': service_id, + 'entity_type': entity_type, 'scope': scope}), 200 + return bp diff --git a/core/enrichment/unmatched.py b/core/enrichment/unmatched.py index 924a3bd9..fdf9fbfb 100644 --- a/core/enrichment/unmatched.py +++ b/core/enrichment/unmatched.py @@ -173,6 +173,46 @@ def build_count_query( return sql, params +# Reset scopes for re-queuing items so the worker re-attempts them. +RESET_SCOPES = ('item', 'failed') + + +def build_reset_query( + service: str, + entity_type: str, + scope: str = 'item', + entity_id=None, +) -> Tuple[str, List]: + """Build the UPDATE that re-queues item(s) for enrichment. + + Re-queuing means clearing ``_match_status`` back to NULL (and + ``_last_attempted`` to NULL): every worker's pending query selects + ``match_status IS NULL`` first, so the item is retried on the next pass. + Nulling last_attempted alone is NOT enough — the not_found retry path uses + ``last_attempted < cutoff`` and ``NULL < cutoff`` is false, so the item + would never be picked up. + + * scope='item' -> a single row (requires entity_id) + * scope='failed' -> every 'not_found' row for this entity type + """ + _validate(service, entity_type) + if scope not in RESET_SCOPES: + raise UnmatchedQueryError(f"Invalid reset scope: {scope!r}") + + meta = _ENTITY_TABLE[entity_type] + table = meta['table'] + ms = match_status_column(service) + la = last_attempted_column(service) + set_clause = f"SET {ms} = NULL, {la} = NULL" + + if scope == 'item': + if not entity_id: + raise UnmatchedQueryError("entity_id is required for an item reset") + return f"UPDATE {table} {set_clause} WHERE id = ?", [entity_id] + # 'failed' — re-queue everything this source explicitly gave up on. + return f"UPDATE {table} {set_clause} WHERE {ms} = 'not_found'", [] + + def build_breakdown_query(service: str, entity_type: str) -> Tuple[str, List]: """Build the matched / not_found / pending / total tally for one entity type.""" _validate(service, entity_type) @@ -211,4 +251,6 @@ __all__ = [ 'build_unmatched_query', 'build_count_query', 'build_breakdown_query', + 'build_reset_query', + 'RESET_SCOPES', ] diff --git a/database/music_database.py b/database/music_database.py index 7dee6199..aceb49c4 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -1079,6 +1079,24 @@ class MusicDatabase: finally: conn.close() + def reset_enrichment(self, service: str, entity_type: str, scope: str = 'item', entity_id=None) -> int: + """Re-queue item(s) for a source by clearing match_status back to NULL. + + scope='item' resets one row (entity_id); scope='failed' resets every + 'not_found' row for that entity type. Returns the number of rows reset. + Raises ``UnmatchedQueryError`` on bad input.""" + from core.enrichment.unmatched import build_reset_query + + sql, params = build_reset_query(service, entity_type, scope, entity_id) + conn = self._get_connection() + try: + cursor = conn.cursor() + cursor.execute(sql, params) + conn.commit() + return cursor.rowcount or 0 + finally: + conn.close() + def _add_mirrored_playlist_explored_column(self, cursor): """Add explored_at column to mirrored_playlists to persist explore badge.""" try: diff --git a/tests/enrichment/test_unmatched.py b/tests/enrichment/test_unmatched.py index 0a0730ab..d6c88f08 100644 --- a/tests/enrichment/test_unmatched.py +++ b/tests/enrichment/test_unmatched.py @@ -17,6 +17,7 @@ from core.enrichment.unmatched import ( UnmatchedQueryError, build_breakdown_query, build_count_query, + build_reset_query, build_unmatched_query, supported_entity_types, ) @@ -154,6 +155,43 @@ def test_db_raises_on_bad_input(db): db.get_enrichment_unmatched('spotify', 'artist', status='bogus') +# -------------------------------------------------------------------------- +# Reset / retry (re-queue) — must clear match_status to NULL so the worker +# re-attempts (nulling last_attempted alone leaves not_found in limbo). +# -------------------------------------------------------------------------- + +def test_reset_builder_requires_id_for_item(): + with pytest.raises(UnmatchedQueryError): + build_reset_query('spotify', 'artist', 'item') # no entity_id + + +def test_reset_builder_bad_scope(): + with pytest.raises(UnmatchedQueryError): + build_reset_query('spotify', 'artist', 'bogus', entity_id='x') + + +def test_reset_builder_nulls_status_not_just_attempted(): + sql, _ = build_reset_query('spotify', 'artist', 'failed') + assert 'spotify_match_status = NULL' in sql + assert 'spotify_last_attempted = NULL' in sql + assert "WHERE spotify_match_status = 'not_found'" in sql + + +def test_reset_item_requeues_to_pending(db): + n = db.reset_enrichment('spotify', 'artist', 'item', entity_id='a2') # was not_found + assert n == 1 + # not_found dropped by 1, pending gained 1 + bd = db.get_enrichment_breakdown('spotify', 'artist') + assert bd == {'matched': 1, 'not_found': 0, 'pending': 2, 'total': 3} + + +def test_reset_failed_requeues_all(db): + n = db.reset_enrichment('spotify', 'album', 'failed') # one not_found album + assert n == 1 + bd = db.get_enrichment_breakdown('spotify', 'album') + assert bd['not_found'] == 0 and bd['pending'] == 1 + + # -------------------------------------------------------------------------- # Flask routes # -------------------------------------------------------------------------- @@ -192,3 +230,24 @@ def test_route_breakdown(client): assert r.status_code == 200 bd = r.get_json()['breakdown'] assert bd['artist'] == {'matched': 1, 'not_found': 1, 'pending': 1, 'total': 3} + + +def test_route_retry_item(client): + r = client.post('/api/enrichment/spotify/retry', + json={'entity_type': 'artist', 'scope': 'item', 'entity_id': 'a2'}) + assert r.status_code == 200 + body = r.get_json() + assert body['success'] is True and body['reset'] == 1 + + +def test_route_retry_failed_bulk(client): + r = client.post('/api/enrichment/spotify/retry', + json={'entity_type': 'artist', 'scope': 'failed'}) + assert r.status_code == 200 + assert r.get_json()['reset'] == 1 # one not_found artist re-queued + + +def test_route_retry_item_missing_id_400(client): + r = client.post('/api/enrichment/spotify/retry', + json={'entity_type': 'artist', 'scope': 'item'}) + assert r.status_code == 400 diff --git a/webui/static/enrichment-manager.js b/webui/static/enrichment-manager.js index f0be70ce..70b4cbac 100644 --- a/webui/static/enrichment-manager.js +++ b/webui/static/enrichment-manager.js @@ -511,15 +511,22 @@ function _emRenderUnmatchedControls() { const data = enrichmentManagerState.unmatched; const supported = (data && data.entity_types) || ['artist']; const total = data ? (data.total || 0) : null; + const entity = enrichmentManagerState.entityTab; + const failed = enrichmentManagerState.breakdown?.[entity]?.not_found || 0; const tabs = supported.map(e => ` `).join(''); + const bulkBtn = failed + ? `` + : ''; host.innerHTML = `
Needs matching ${total == null ? '' : `${total.toLocaleString()}`} + ${bulkBtn}
${tabs}
@@ -535,7 +542,8 @@ function _emRenderUnmatchedControls() { oninput="onEnrichmentSearchInput(this.value)">
-
`; + +
Failed lookups auto-retry after 30 days · “Retry” re-queues immediately · “Match” assigns a result by hand.
`; } function _emRenderUnmatchedList() { @@ -671,13 +679,15 @@ async function toggleEnrichmentWorker(id) { async function retryEnrichmentItem(service, entityType, entityId, btn) { if (btn) { btn.disabled = true; btn.textContent = '…'; } try { - const res = await fetch('/api/library/clear-match', { - method: 'PUT', + // Reset match_status to NULL (pending) so the worker re-attempts on its + // next pass — see /retry (clearing to not_found would NOT re-queue). + const res = await fetch(`/api/enrichment/${service}/retry`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ entity_type: entityType, entity_id: entityId, service }), + body: JSON.stringify({ entity_type: entityType, scope: 'item', entity_id: entityId }), }); const data = await res.json().catch(() => ({})); - if (data.success) { + if (res.ok && data.success) { showToast('Re-queued for enrichment', 'success'); await Promise.all([_emLoadBreakdown(service), _emLoadUnmatched()]); _emRenderStats(); @@ -692,6 +702,39 @@ async function retryEnrichmentItem(service, entityType, entityId, btn) { } } +// Bulk: re-queue every not_found item of the current entity type. +async function retryAllFailedEnrichment(btn) { + const service = enrichmentManagerState.selected; + const entity = enrichmentManagerState.entityTab; + const bd = enrichmentManagerState.breakdown?.[entity]; + const failed = bd ? (bd.not_found || 0) : 0; + if (!failed) { showToast('No failed items to retry', 'info'); return; } + if (!confirm(`Re-queue all ${failed.toLocaleString()} not-found ${_emEntityLabel(entity, true).toLowerCase()} for ${_emWorkerById[service].name}? The worker will retry them on its next pass.`)) return; + if (btn) { btn.disabled = true; btn.textContent = 'Re-queuing…'; } + try { + const res = await fetch(`/api/enrichment/${service}/retry`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ entity_type: entity, scope: 'failed' }), + }); + const data = await res.json().catch(() => ({})); + if (res.ok && data.success) { + showToast(`Re-queued ${(data.reset || 0).toLocaleString()} item(s)`, 'success'); + enrichmentManagerState.page = 0; + await Promise.all([_emLoadBreakdown(service), _emLoadUnmatched()]); + _emRenderStats(); + _emRenderUnmatchedControls(); + _emRenderUnmatchedList(); + } else { + showToast(data.error || 'Failed to re-queue', 'error'); + } + } catch (_e) { + showToast('Error re-queuing failed items', 'error'); + } finally { + if (btn) { btn.disabled = false; btn.textContent = '↻ Retry all failed'; } + } +} + // ── Inline manual match (decoupled from the library artist-detail page) ─────── function openEnrichmentMatch(service, entityType, entityId, anchorBtn) { @@ -813,4 +856,5 @@ window.onEnrichmentSearchInput = onEnrichmentSearchInput; window.changeEnrichmentPage = changeEnrichmentPage; window.toggleEnrichmentWorker = toggleEnrichmentWorker; window.retryEnrichmentItem = retryEnrichmentItem; +window.retryAllFailedEnrichment = retryAllFailedEnrichment; window.openEnrichmentMatch = openEnrichmentMatch; diff --git a/webui/static/style.css b/webui/static/style.css index 0fa81c2a..c5616812 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -65129,3 +65129,8 @@ body.em-scroll-lock { overflow: hidden; } .em-search-wrap .em-search { padding-left: 30px; } .em-search:focus, .em-select:focus { outline: none; border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.6); } .em-row:hover { background: rgba(var(--em-accent-rgb, 255,255,255), 0.07); border-color: rgba(var(--em-accent-rgb, 255,255,255), 0.14); } + +/* Bulk retry-all + helper hint */ +.em-retry-all { margin-left: 4px; } +.em-hint { font-size: 11px; color: rgba(255,255,255,0.38); flex: 0 0 auto; margin-top: -2px; } +.em-hint :is(b, strong) { color: rgba(255,255,255,0.6); } From e53a157793cbb4f7bb2e9499c182675260a82173 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 19:45:04 -0700 Subject: [PATCH 025/108] Enrichment manager: 'process this group first' + refined hero header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-worker processing-order override + UI polish. Feature — pin an entity group to enrich first: - Each worker normally runs artist -> album -> track. A user can pin one group (artist/album/track) to run first from the modal; the worker keeps that group first until it's exhausted, then resumes the normal chain. - core/worker_utils.py: read_enrichment_priority() (reads _enrichment_priority each loop, live) + priority_pending_item() (shared, whitelisted query returning the worker's expected item shape; Spotify/iTunes get album_individual/track_individual via a type map). - A guarded ~6-line hook at the top of all 11 workers' _get_next_item. CRITICAL: when nothing is pinned (default) the hook returns immediately, so default enrichment order is byte-identical to before. Discogs (no track) and Genius (no album) only honor their supported entities. - core/enrichment/api.py: GET/POST /api/enrichment//priority (+ config_get hook); POST validates the entity against what the source enriches. - 14 new tests (helper shapes, exhaustion, route get/set/clear/validate). UI: - Refined hero header: identity + inline status left, single Pause right, 'now enriching' quiet sub-line; overall coverage % moved into the stats section ('82% matched · 1,203 of 1,460'). Hero gently pulses while running. - New processing-order strip: artist→album→track steps showing the live phase (pulsing 'now'), pinned group ('first' + 📌), and done/remaining; click a step to pin it, click again for auto. py_compile clean across all 11 workers; 52 enrichment tests green. --- core/amazon_worker.py | 10 ++ core/audiodb_worker.py | 10 ++ core/deezer_worker.py | 10 ++ core/discogs_worker.py | 10 ++ core/enrichment/api.py | 49 +++++++- core/genius_worker.py | 10 ++ core/itunes_worker.py | 11 ++ core/lastfm_worker.py | 10 ++ core/musicbrainz_worker.py | 10 ++ core/qobuz_worker.py | 10 ++ core/spotify_worker.py | 11 ++ core/tidal_worker.py | 10 ++ core/worker_utils.py | 64 +++++++++++ tests/enrichment/test_worker_priority.py | 138 +++++++++++++++++++++++ web_server.py | 1 + webui/static/enrichment-manager.js | 136 +++++++++++++++++++--- webui/static/style.css | 50 +++++++- 17 files changed, 527 insertions(+), 23 deletions(-) create mode 100644 tests/enrichment/test_worker_priority.py diff --git a/core/amazon_worker.py b/core/amazon_worker.py index c4dbc0c7..ee4b9751 100644 --- a/core/amazon_worker.py +++ b/core/amazon_worker.py @@ -174,6 +174,16 @@ class AmazonWorker: cursor = conn.cursor() self._ensure_amazon_schema(cursor) + # Pinned-group override (Manage Enrichment Workers): process one + # entity type first, then fall through to the normal chain. Unset or + # exhausted ⇒ default artist→album→track order, unchanged. + from core.worker_utils import read_enrichment_priority, priority_pending_item + _prio = read_enrichment_priority('amazon') + if _prio: + _pi = priority_pending_item(cursor, 'amazon', _prio) + if _pi: + return _pi + # Priority 1: Unattempted artists cursor.execute(""" SELECT id, name FROM artists diff --git a/core/audiodb_worker.py b/core/audiodb_worker.py index ec687346..a8767cb8 100644 --- a/core/audiodb_worker.py +++ b/core/audiodb_worker.py @@ -162,6 +162,16 @@ class AudioDBWorker: conn = self.db._get_connection() cursor = conn.cursor() + # Pinned-group override (Manage Enrichment Workers): process one + # entity type first, then fall through to the normal chain. Unset or + # exhausted ⇒ default artist→album→track order, unchanged. + from core.worker_utils import read_enrichment_priority, priority_pending_item + _prio = read_enrichment_priority('audiodb') + if _prio: + _pi = priority_pending_item(cursor, 'audiodb', _prio) + if _pi: + return _pi + # Priority 1: Unattempted artists cursor.execute(""" SELECT id, name diff --git a/core/deezer_worker.py b/core/deezer_worker.py index aba5c6fb..9b592627 100644 --- a/core/deezer_worker.py +++ b/core/deezer_worker.py @@ -163,6 +163,16 @@ class DeezerWorker: conn = self.db._get_connection() cursor = conn.cursor() + # Pinned-group override (Manage Enrichment Workers): process one + # entity type first, then fall through to the normal chain. Unset or + # exhausted ⇒ default artist→album→track order, unchanged. + from core.worker_utils import read_enrichment_priority, priority_pending_item + _prio = read_enrichment_priority('deezer') + if _prio: + _pi = priority_pending_item(cursor, 'deezer', _prio) + if _pi: + return _pi + # Priority 1: Unattempted artists cursor.execute(""" SELECT id, name diff --git a/core/discogs_worker.py b/core/discogs_worker.py index 8663934f..429ba320 100644 --- a/core/discogs_worker.py +++ b/core/discogs_worker.py @@ -174,6 +174,16 @@ class DiscogsWorker: conn = self.db._get_connection() cursor = conn.cursor() + # Pinned-group override (Manage Enrichment Workers): process one + # entity type first, then fall through to the normal chain. Discogs + # has no track endpoint, so only artist/album are honored. + from core.worker_utils import read_enrichment_priority, priority_pending_item + _prio = read_enrichment_priority('discogs') + if _prio in ('artist', 'album'): + _pi = priority_pending_item(cursor, 'discogs', _prio) + if _pi: + return _pi + # Priority 1: Unattempted artists cursor.execute(""" SELECT id, name FROM artists diff --git a/core/enrichment/api.py b/core/enrichment/api.py index 746e0bd5..99c26110 100644 --- a/core/enrichment/api.py +++ b/core/enrichment/api.py @@ -35,6 +35,7 @@ logger = get_logger("enrichment.api") # Hooks the host wires up so the blueprint can persist pause state and # clean up auto-pause / yield-override sets without circular imports. _config_set: Optional[Callable[[str, Any], None]] = None +_config_get: Optional[Callable[[str, Any], Any]] = None _auto_paused_discard: Optional[Callable[[str], None]] = None _yield_override_add: Optional[Callable[[str], None]] = None _db_getter: Optional[Callable[[], Any]] = None @@ -43,6 +44,7 @@ _db_getter: Optional[Callable[[], Any]] = None def configure( *, config_set: Optional[Callable[[str, Any], None]] = None, + config_get: Optional[Callable[[str, Any], Any]] = None, auto_paused_discard: Optional[Callable[[str], None]] = None, yield_override_add: Optional[Callable[[str], None]] = None, db_getter: Optional[Callable[[], Any]] = None, @@ -51,10 +53,12 @@ def configure( Each is optional — pass None for hosts that don't have a corresponding mechanism (e.g. tests). ``db_getter`` returns the live ``MusicDatabase`` - for the unmatched-browser routes. + for the unmatched-browser routes; ``config_get``/``config_set`` read and + write the per-worker priority override. """ - global _config_set, _auto_paused_discard, _yield_override_add, _db_getter + global _config_set, _config_get, _auto_paused_discard, _yield_override_add, _db_getter _config_set = config_set + _config_get = config_get _auto_paused_discard = auto_paused_discard _yield_override_add = yield_override_add _db_getter = db_getter @@ -250,4 +254,45 @@ def create_blueprint() -> Blueprint: return jsonify({'success': True, 'reset': count, 'service': service_id, 'entity_type': entity_type, 'scope': scope}), 200 + @bp.route('/api/enrichment//priority', methods=['GET']) + def enrichment_get_priority(service_id: str): + """Return the pinned 'process this group first' entity for a worker.""" + if service_id not in SERVICE_ENTITY_SUPPORT: + return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404 + priority = '' + if _config_get is not None: + try: + priority = (_config_get(f'{service_id}_enrichment_priority', '') or '').strip().lower() + except Exception as e: + logger.debug("reading %s priority: %s", service_id, e) + if priority not in supported_entity_types(service_id): + priority = '' + return jsonify({'service': service_id, 'priority': priority, + 'entity_types': list(supported_entity_types(service_id))}), 200 + + @bp.route('/api/enrichment//priority', methods=['POST']) + def enrichment_set_priority(service_id: str): + """Pin (or clear) the entity type the worker should process first. + + Body: ``entity`` = 'artist'|'album'|'track' to pin, or '' / null / 'none' + to clear. Must be an entity type the source actually enriches. + """ + if service_id not in SERVICE_ENTITY_SUPPORT: + return jsonify({'error': f'Unknown enrichment service: {service_id}'}), 404 + if _config_set is None: + return jsonify({'error': 'config unavailable'}), 503 + data = request.get_json(silent=True) or {} + entity = (data.get('entity') or '').strip().lower() + if entity in ('none', 'clear'): + entity = '' + if entity and entity not in supported_entity_types(service_id): + return jsonify({'error': f'{service_id} does not enrich {entity!r}'}), 400 + try: + _config_set(f'{service_id}_enrichment_priority', entity) + except Exception as e: + logger.error("setting %s priority: %s", service_id, e) + return jsonify({'error': str(e)}), 500 + logger.info("%s enrichment priority set to %r via UI", service_id, entity or '(none)') + return jsonify({'success': True, 'service': service_id, 'priority': entity}), 200 + return bp diff --git a/core/genius_worker.py b/core/genius_worker.py index e2e731fd..1184e6db 100644 --- a/core/genius_worker.py +++ b/core/genius_worker.py @@ -178,6 +178,16 @@ class GeniusWorker: conn = self.db._get_connection() cursor = conn.cursor() + # Pinned-group override (Manage Enrichment Workers): process one + # entity type first, then fall through to the normal chain. Genius + # is artist/track only, so albums are not honored. + from core.worker_utils import read_enrichment_priority, priority_pending_item + _prio = read_enrichment_priority('genius') + if _prio in ('artist', 'track'): + _pi = priority_pending_item(cursor, 'genius', _prio) + if _pi: + return _pi + # Priority 1: Unattempted artists cursor.execute(""" SELECT id, name diff --git a/core/itunes_worker.py b/core/itunes_worker.py index 50cf0df0..cca60db0 100644 --- a/core/itunes_worker.py +++ b/core/itunes_worker.py @@ -172,6 +172,17 @@ class iTunesWorker: conn = self.db._get_connection() cursor = conn.cursor() + # Pinned-group override (Manage Enrichment Workers): process one + # entity type first, then fall through to the normal chain. Unset or + # exhausted ⇒ default artist→album→track order, unchanged. + from core.worker_utils import read_enrichment_priority, priority_pending_item + _prio = read_enrichment_priority('itunes') + if _prio: + _pi = priority_pending_item(cursor, 'itunes', _prio, + {'album': 'album_individual', 'track': 'track_individual'}) + if _pi: + return _pi + # Priority 1: Unattempted artists cursor.execute(""" SELECT id, name diff --git a/core/lastfm_worker.py b/core/lastfm_worker.py index 69a4c360..c2011e40 100644 --- a/core/lastfm_worker.py +++ b/core/lastfm_worker.py @@ -177,6 +177,16 @@ class LastFMWorker: conn = self.db._get_connection() cursor = conn.cursor() + # Pinned-group override (Manage Enrichment Workers): process one + # entity type first, then fall through to the normal chain. Unset or + # exhausted ⇒ default artist→album→track order, unchanged. + from core.worker_utils import read_enrichment_priority, priority_pending_item + _prio = read_enrichment_priority('lastfm') + if _prio: + _pi = priority_pending_item(cursor, 'lastfm', _prio) + if _pi: + return _pi + # Priority 1: Unattempted artists cursor.execute(""" SELECT id, name diff --git a/core/musicbrainz_worker.py b/core/musicbrainz_worker.py index bc494433..4f3adf55 100644 --- a/core/musicbrainz_worker.py +++ b/core/musicbrainz_worker.py @@ -166,6 +166,16 @@ class MusicBrainzWorker: conn = self.db._get_connection() cursor = conn.cursor() + # Pinned-group override (Manage Enrichment Workers): process one + # entity type first, then fall through to the normal chain. Unset or + # exhausted ⇒ default artist→album→track order, unchanged. + from core.worker_utils import read_enrichment_priority, priority_pending_item + _prio = read_enrichment_priority('musicbrainz') + if _prio: + _pi = priority_pending_item(cursor, 'musicbrainz', _prio) + if _pi: + return _pi + # Priority 1: Unattempted artists cursor.execute(""" SELECT id, name diff --git a/core/qobuz_worker.py b/core/qobuz_worker.py index c6ba2be7..706d6295 100644 --- a/core/qobuz_worker.py +++ b/core/qobuz_worker.py @@ -186,6 +186,16 @@ class QobuzWorker: conn = self.db._get_connection() cursor = conn.cursor() + # Pinned-group override (Manage Enrichment Workers): process one + # entity type first, then fall through to the normal chain. Unset or + # exhausted ⇒ default artist→album→track order, unchanged. + from core.worker_utils import read_enrichment_priority, priority_pending_item + _prio = read_enrichment_priority('qobuz') + if _prio: + _pi = priority_pending_item(cursor, 'qobuz', _prio) + if _pi: + return _pi + # Priority 1: Unattempted artists cursor.execute(""" SELECT id, name diff --git a/core/spotify_worker.py b/core/spotify_worker.py index 9349bb74..4e0b3e66 100644 --- a/core/spotify_worker.py +++ b/core/spotify_worker.py @@ -260,6 +260,17 @@ class SpotifyWorker: conn = self.db._get_connection() cursor = conn.cursor() + # Pinned-group override (Manage Enrichment Workers): process one + # entity type first, then fall through to the normal chain. Unset or + # exhausted ⇒ default artist→album→track order, unchanged. + from core.worker_utils import read_enrichment_priority, priority_pending_item + _prio = read_enrichment_priority('spotify') + if _prio: + _pi = priority_pending_item(cursor, 'spotify', _prio, + {'album': 'album_individual', 'track': 'track_individual'}) + if _pi: + return _pi + # Priority 1: Unattempted artists cursor.execute(""" SELECT id, name diff --git a/core/tidal_worker.py b/core/tidal_worker.py index 4bcb434a..051f2843 100644 --- a/core/tidal_worker.py +++ b/core/tidal_worker.py @@ -198,6 +198,16 @@ class TidalWorker: conn = self.db._get_connection() cursor = conn.cursor() + # Pinned-group override (Manage Enrichment Workers): process one + # entity type first, then fall through to the normal chain. Unset or + # exhausted ⇒ default artist→album→track order, unchanged. + from core.worker_utils import read_enrichment_priority, priority_pending_item + _prio = read_enrichment_priority('tidal') + if _prio: + _pi = priority_pending_item(cursor, 'tidal', _prio) + if _pi: + return _pi + # Priority 1: Unattempted artists cursor.execute(""" SELECT id, name diff --git a/core/worker_utils.py b/core/worker_utils.py index e572c702..a8d965b9 100644 --- a/core/worker_utils.py +++ b/core/worker_utils.py @@ -78,3 +78,67 @@ def set_album_api_track_count(cursor, album_id, count): logger.warning( "Failed to cache api_track_count for album %s: %s", album_id, e ) + + +# --- Enrichment "process this group first" override ----------------------- +# Each enrichment worker normally processes artist -> album -> track. A user +# can pin one entity type to run first via the Manage Enrichment Workers modal; +# the choice is stored in config as "_enrichment_priority" and read +# at the top of each worker's _get_next_item so it takes effect live. When the +# pinned group is exhausted (or unset), the worker falls back to its normal +# chain — so the default path is unchanged. + +PRIORITY_ENTITIES = ('artist', 'album', 'track') + + +def read_enrichment_priority(service: str) -> str: + """Return the pinned entity ('artist'|'album'|'track') for a worker, or ''. + + Read every loop so the override applies without restarting the worker. + Any error / unset / invalid value yields '' (no override).""" + try: + from config.settings import config_manager + val = (config_manager.get(f'{service}_enrichment_priority', '') or '') + val = str(val).strip().lower() + return val if val in PRIORITY_ENTITIES else '' + except Exception: + return '' + + +def priority_pending_item(cursor, service, entity, type_overrides=None): + """Return one pending (NULL match_status) item of `entity`, or None. + + `service` is the column prefix (e.g. 'spotify' -> spotify_match_status) and + MUST be a trusted worker-supplied literal (it is interpolated into SQL). + `type_overrides` maps the canonical entity to the worker's dispatch 'type' + string — Spotify/iTunes process individual items as 'album_individual' / + 'track_individual', the other workers use 'album' / 'track'. The returned + dict matches the shape those workers already return from _get_next_item.""" + if not str(service).isalpha() or entity not in PRIORITY_ENTITIES: + return None + type_overrides = type_overrides or {} + ms = f"{service}_match_status" + + if entity == 'artist': + cursor.execute( + f"SELECT id, name FROM artists WHERE {ms} IS NULL AND id IS NOT NULL " + f"ORDER BY id ASC LIMIT 1" + ) + r = cursor.fetchone() + return {'type': type_overrides.get('artist', 'artist'), 'id': r[0], 'name': r[1]} if r else None + + if entity == 'album': + cursor.execute( + f"SELECT a.id, a.title, ar.name FROM albums a JOIN artists ar ON a.artist_id = ar.id " + f"WHERE a.{ms} IS NULL AND a.id IS NOT NULL ORDER BY a.id ASC LIMIT 1" + ) + r = cursor.fetchone() + return {'type': type_overrides.get('album', 'album'), 'id': r[0], 'name': r[1], 'artist': r[2]} if r else None + + # track + cursor.execute( + f"SELECT t.id, t.title, ar.name FROM tracks t JOIN artists ar ON t.artist_id = ar.id " + f"WHERE t.{ms} IS NULL AND t.id IS NOT NULL ORDER BY t.id ASC LIMIT 1" + ) + r = cursor.fetchone() + return {'type': type_overrides.get('track', 'track'), 'id': r[0], 'name': r[1], 'artist': r[2]} if r else None diff --git a/tests/enrichment/test_worker_priority.py b/tests/enrichment/test_worker_priority.py new file mode 100644 index 00000000..e17e5582 --- /dev/null +++ b/tests/enrichment/test_worker_priority.py @@ -0,0 +1,138 @@ +"""Priority 'process this group first' helper for enrichment workers. + +The shared helper returns one pending item of a chosen entity type in the +shape the worker's dispatch already expects (with Spotify/iTunes mapped to +their album_individual / track_individual types). Default path (no override) +is exercised by the workers themselves and unchanged. +""" + +from __future__ import annotations + +import pytest + +from core.worker_utils import ( + PRIORITY_ENTITIES, + priority_pending_item, + read_enrichment_priority, +) +from database.music_database import MusicDatabase + + +@pytest.fixture +def db(tmp_path): + d = MusicDatabase(str(tmp_path / 'prio.db')) + conn = d._get_connection() + cur = conn.cursor() + cur.execute("INSERT INTO artists (id, name) VALUES ('a1', 'Pending Artist')") # NULL status + cur.execute("INSERT INTO artists (id, name, spotify_match_status) VALUES ('a2','Done','matched')") + cur.execute("INSERT INTO albums (id, artist_id, title) VALUES ('al1','a2','Pending Album')") # NULL status + cur.execute("INSERT INTO tracks (id, album_id, artist_id, title) VALUES ('t1','al1','a2','Pending Track')") + conn.commit() + conn.close() + return d + + +def _cur(db): + return db._get_connection().cursor() + + +def test_priority_artist_shape(db): + item = priority_pending_item(_cur(db), 'spotify', 'artist') + assert item == {'type': 'artist', 'id': 'a1', 'name': 'Pending Artist'} + + +def test_priority_album_default_type(db): + item = priority_pending_item(_cur(db), 'spotify', 'album') + assert item['id'] == 'al1' and item['name'] == 'Pending Album' and item['artist'] == 'Done' + assert item['type'] == 'album' # default type string + + +def test_priority_album_type_override_for_spotify_itunes(db): + item = priority_pending_item(_cur(db), 'spotify', 'album', + {'album': 'album_individual', 'track': 'track_individual'}) + assert item['type'] == 'album_individual' # matches Spotify/iTunes dispatch + + +def test_priority_track_shape(db): + item = priority_pending_item(_cur(db), 'spotify', 'track') + assert item['id'] == 't1' and item['type'] == 'track' and item['artist'] == 'Done' + + +def test_priority_returns_none_when_group_exhausted(db): + # No pending artists once a1 is matched -> None, so worker resumes its chain. + conn = db._get_connection(); conn.execute("UPDATE artists SET spotify_match_status='matched' WHERE id='a1'"); conn.commit(); conn.close() + assert priority_pending_item(_cur(db), 'spotify', 'artist') is None + + +def test_priority_rejects_bad_entity_and_service(db): + assert priority_pending_item(_cur(db), 'spotify', 'bogus') is None + assert priority_pending_item(_cur(db), 'spot;drop', 'artist') is None # non-alpha service blocked + + +def test_read_priority_unset_is_empty(): + # Unknown/unset key -> '' (no override). Uses the real config_manager. + assert read_enrichment_priority('definitely_not_a_service') == '' + + +def test_read_priority_roundtrip(): + from config.settings import config_manager + key = 'spotify_enrichment_priority' + old = config_manager.get(key, '') + try: + config_manager.set(key, 'album') + assert read_enrichment_priority('spotify') == 'album' + config_manager.set(key, 'bogus') + assert read_enrichment_priority('spotify') == '' # invalid -> ignored + finally: + config_manager.set(key, old) + + +def test_priority_entities_constant(): + assert PRIORITY_ENTITIES == ('artist', 'album', 'track') + + +# --- priority GET/POST routes --------------------------------------------- + +@pytest.fixture +def client(): + from flask import Flask + from core.enrichment import api as enrichment_api + store = {} + enrichment_api.configure( + config_get=lambda k, d=None: store.get(k, d), + config_set=lambda k, v: store.__setitem__(k, v), + db_getter=lambda: None, + ) + app = Flask(__name__) + app.register_blueprint(enrichment_api.create_blueprint()) + with app.test_client() as c: + c._store = store + yield c + enrichment_api.configure(config_get=None, config_set=None, db_getter=None) + + +def test_route_priority_get_default_empty(client): + r = client.get('/api/enrichment/spotify/priority') + assert r.status_code == 200 + assert r.get_json()['priority'] == '' + + +def test_route_priority_set_and_get(client): + assert client.post('/api/enrichment/spotify/priority', json={'entity': 'album'}).status_code == 200 + assert client._store['spotify_enrichment_priority'] == 'album' + assert client.get('/api/enrichment/spotify/priority').get_json()['priority'] == 'album' + + +def test_route_priority_clear(client): + client.post('/api/enrichment/spotify/priority', json={'entity': 'album'}) + client.post('/api/enrichment/spotify/priority', json={'entity': 'none'}) + assert client.get('/api/enrichment/spotify/priority').get_json()['priority'] == '' + + +def test_route_priority_rejects_unsupported_entity(client): + # Genius has no albums -> 400 + assert client.post('/api/enrichment/genius/priority', json={'entity': 'album'}).status_code == 400 + + +def test_route_priority_unknown_service_404(client): + assert client.get('/api/enrichment/bogus/priority').status_code == 404 diff --git a/web_server.py b/web_server.py index 6d0d007e..8d8caa8d 100644 --- a/web_server.py +++ b/web_server.py @@ -34643,6 +34643,7 @@ _register_enrichment_services([ _configure_enrichment_api( config_set=lambda key, value: config_manager.set(key, value), + config_get=lambda key, default=None: config_manager.get(key, default), auto_paused_discard=lambda token: _download_auto_paused.discard(token), yield_override_add=lambda token: _download_yield_override.add(token), db_getter=get_database, diff --git a/webui/static/enrichment-manager.js b/webui/static/enrichment-manager.js index 70b4cbac..563111f1 100644 --- a/webui/static/enrichment-manager.js +++ b/webui/static/enrichment-manager.js @@ -76,6 +76,7 @@ const enrichmentManagerState = { selected: null, statuses: {}, // id -> last /status payload breakdown: null, // selected worker's breakdown + priority: '', // pinned 'process first' entity for selected worker ('' = auto) entityTab: 'artist', statusFilter: 'unmatched', search: '', @@ -229,6 +230,7 @@ async function _emPollSelected() { enrichmentManagerState.statuses[id] = await res.json(); _emUpdateHeaderLive(); // in-place — no logo reflow/flicker _emUpdateRailRow(id); + _emRenderChain(); // current-phase highlight tracks live } } catch (_e) { /* transient — keep last */ } } @@ -307,6 +309,7 @@ async function selectEnrichmentWorker(id) { enrichmentManagerState.selected = id; enrichmentManagerState.breakdown = null; enrichmentManagerState.unmatched = null; + enrichmentManagerState.priority = ''; enrichmentManagerState.search = ''; enrichmentManagerState.page = 0; enrichmentManagerState.statusFilter = 'unmatched'; @@ -316,10 +319,29 @@ async function selectEnrichmentWorker(id) { // unmatched call returns entity_types; default to artist meanwhile). enrichmentManagerState.entityTab = 'artist'; renderEnrichmentPanel(); - await Promise.all([_emLoadBreakdown(id), _emLoadUnmatched()]); + await Promise.all([_emLoadBreakdown(id), _emLoadUnmatched(), _emLoadPriority(id)]); renderEnrichmentPanel(); } +async function _emLoadPriority(id) { + try { + const res = await fetch(`/api/enrichment/${id}/priority`); + enrichmentManagerState.priority = res.ok ? ((await res.json()).priority || '') : ''; + } catch (_e) { + enrichmentManagerState.priority = ''; + } +} + +// Which phase the worker is on right now, from current_item.type. +function _emCurrentPhase(status) { + const t = status && status.current_item && status.current_item.type; + if (!t) return ''; + if (t.indexOf('artist') === 0) return 'artist'; + if (t.indexOf('album') === 0) return 'album'; + if (t.indexOf('track') === 0) return 'track'; + return ''; +} + async function _emLoadBreakdown(id) { try { const res = await fetch(`/api/enrichment/${id}/breakdown`); @@ -367,7 +389,11 @@ function renderEnrichmentPanel() { panel.innerHTML = `
-
Enrichment coverage
+
+
+ Enrichment coverage + +
@@ -375,11 +401,81 @@ function renderEnrichmentPanel() {
`; _emRenderPanelHeader(); + _emRenderChain(); _emRenderStats(); _emRenderUnmatchedControls(); _emRenderUnmatchedList(); } +// Processing-order strip: shows the artist→album→track chain, where the worker +// currently is, and lets the user pin one group to run first. +function _emRenderChain() { + const host = document.getElementById('em-chain'); + if (!host) return; + const id = enrichmentManagerState.selected; + const supported = (enrichmentManagerState.unmatched && enrichmentManagerState.unmatched.entity_types) + || (enrichmentManagerState.breakdown && Object.keys(enrichmentManagerState.breakdown)) + || ['artist']; + const status = enrichmentManagerState.statuses[id]; + const phase = _emCurrentPhase(status); + const pinned = enrichmentManagerState.priority; + const bd = enrichmentManagerState.breakdown || {}; + const glyphs = { artist: '🎤', album: '💿', track: '🎵' }; + + const steps = supported.map((e, i) => { + const d = bd[e] || {}; + const pending = (d.pending || 0) + (d.not_found || 0); + const isCurrent = phase === e; + const isPinned = pinned === e; + const isDone = bd[e] && pending === 0; + const cls = ['em-step', + isPinned ? 'em-step--pinned' : '', + isCurrent ? 'em-step--current' : '', + isDone ? 'em-step--done' : ''].filter(Boolean).join(' '); + const note = isPinned ? 'first' : (isCurrent ? 'now' : (isDone ? 'done' : `${pending.toLocaleString()} left`)); + const arrow = i < supported.length - 1 ? '→' : ''; + return ` + ${arrow}`; + }).join(''); + + host.innerHTML = ` +
+ Processing order + ${pinned + ? `Pinned ${_emEntityLabel(pinned, true)} first · click again for auto` + : 'Click a group to process it first'} +
+
${steps}
`; +} + +async function setEnrichmentPriority(entity) { + const id = enrichmentManagerState.selected; + try { + const res = await fetch(`/api/enrichment/${id}/priority`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ entity: entity || 'none' }), + }); + const data = await res.json().catch(() => ({})); + if (res.ok && data.success) { + enrichmentManagerState.priority = data.priority || ''; + showToast(enrichmentManagerState.priority + ? `${_emWorkerById[id].name} will process ${_emEntityLabel(enrichmentManagerState.priority, true).toLowerCase()} first` + : `${_emWorkerById[id].name} back to automatic order`, 'success'); + _emRenderChain(); + } else { + showToast(data.error || 'Could not set processing order', 'error'); + } + } catch (_e) { + showToast('Error setting processing order', 'error'); + } +} + function _emRenderPanelHeader() { const host = document.getElementById('em-panel-header'); if (!host) return; @@ -393,14 +489,15 @@ function _emRenderPanelHeader() {
${_emIconHtml(id, 'lg')}
-
${_emEscape(worker.name)} enrichment
+
+ ${_emEscape(worker.name)} enrichment + +
-
- - +
`; @@ -415,14 +512,8 @@ function _emUpdateHeaderLive() { const pill = document.getElementById('em-ph-pill'); if (pill) { pill.className = `em-pill em-pill--${info.cls}`; pill.textContent = info.label; } - const metric = document.getElementById('em-ph-metric'); - if (metric) { - const pct = _emOverallPct(status); - metric.innerHTML = pct == null - ? '' - : `${pct}% - enriched`; - } + const hero = document.querySelector('#em-panel-header .em-hero'); + if (hero) hero.classList.toggle('em-hero--live', info.cls === 'running'); const cur = document.getElementById('em-ph-current'); if (cur) { @@ -497,6 +588,18 @@ function _emRenderStats() { `; }).join(''); + // Overall matched coverage across every entity type (relocated here from + // the hero per the refined-banner header). + const overall = document.getElementById('em-coverage-overall'); + if (overall) { + let m = 0, t = 0; + Object.values(bd).forEach(d => { m += d.matched || 0; t += d.total || 0; }); + const pct = t ? Math.round((m / t) * 100) : 0; + overall.innerHTML = t + ? `${pct}% matched · ${m.toLocaleString()} of ${t.toLocaleString()}` + : ''; + } + // Animate the segments in from 0 on the next frame (CSS transition does the rest). requestAnimationFrame(() => { host.querySelectorAll('.em-seg-fill').forEach(el => { @@ -746,11 +849,11 @@ function openEnrichmentMatch(service, entityType, entityId, anchorBtn) { const overlay = document.createElement('div'); overlay.id = 'enrichment-match-overlay'; - overlay.className = 'modal-overlay'; + overlay.className = 'modal-overlay em-overlay'; overlay.style.zIndex = '10010'; // above the manager modal overlay.onclick = (e) => { if (e.target === overlay) overlay.remove(); }; overlay.innerHTML = ` -
+

Match ${_emEntityLabel(entityType)} on ${_emEscape(_emWorkerById[service]?.name || service)}

@@ -855,6 +958,7 @@ window.setEnrichmentStatusFilter = setEnrichmentStatusFilter; window.onEnrichmentSearchInput = onEnrichmentSearchInput; window.changeEnrichmentPage = changeEnrichmentPage; window.toggleEnrichmentWorker = toggleEnrichmentWorker; +window.setEnrichmentPriority = setEnrichmentPriority; window.retryEnrichmentItem = retryEnrichmentItem; window.retryAllFailedEnrichment = retryAllFailedEnrichment; window.openEnrichmentMatch = openEnrichmentMatch; diff --git a/webui/static/style.css b/webui/static/style.css index c5616812..22b2de9a 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -65000,7 +65000,7 @@ body.em-scroll-lock { overflow: hidden; } from { opacity: 0; transform: translateY(14px) scale(0.97); } to { opacity: 1; transform: translateY(0) scale(1); } } -.enrichment-manager-modal.em-in { animation: em-pop-in 0.28s cubic-bezier(0.16, 1, 0.3, 1) both; } +.em-in { animation: em-pop-in 0.28s cubic-bezier(0.16, 1, 0.3, 1) both; } .em-overlay.em-closing .enrichment-manager-modal { transform: scale(0.98); opacity: 0; transition: all 0.16s ease; } .enrichment-manager-modal:focus { outline: none; } @@ -65083,14 +65083,54 @@ body.em-scroll-lock { overflow: hidden; } background: radial-gradient(circle, rgba(var(--em-accent-rgb, 99,102,241), 0.22), transparent 70%); pointer-events: none; } +/* Gentle 'alive' pulse while the worker is actively running. */ +.em-hero--live .em-hero-glow { animation: em-glow-pulse 3.2s ease-in-out infinite; } +@keyframes em-glow-pulse { 0%,100% { opacity: 0.55; } 50% { opacity: 1; } } +@media (prefers-reduced-motion: reduce) { .em-hero--live .em-hero-glow { animation: none; } } .em-hero .em-icon { width: 56px; height: 56px; } .em-hero .em-ph-titles { flex: 1 1 auto; min-width: 0; z-index: 1; } +.em-ph-nameline { display: flex; align-items: center; gap: 11px; flex-wrap: wrap; } .em-ph-name-sub { font-size: 14px; font-weight: 500; color: rgba(255,255,255,0.45); } -.em-hero-metric { display: flex; flex-direction: column; align-items: center; justify-content: center; line-height: 1; z-index: 1; padding: 0 6px; } -.em-hero-pct { font-size: 30px; font-weight: 800; color: #fff; } -.em-hero-pct-sym { font-size: 16px; opacity: 0.6; } -.em-hero-pct-label { font-size: 10px; text-transform: uppercase; letter-spacing: 0.6px; color: rgba(255,255,255,0.45); margin-top: 3px; } +.em-hero .em-ph-actions { z-index: 1; } +/* Processing-order chain strip */ +.em-chain { display: flex; flex-direction: column; gap: 9px; flex: 0 0 auto; } +.em-chain-head { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; flex-wrap: wrap; } +.em-chain-hint { font-size: 11.5px; color: rgba(255,255,255,0.4); } +.em-chain-hint strong { color: rgb(var(--em-accent-rgb, 129,140,248)); } +.em-chain-flow { display: flex; align-items: stretch; gap: 8px; flex-wrap: wrap; } +.em-step { + flex: 1 1 0; min-width: 120px; + display: flex; flex-direction: column; align-items: flex-start; gap: 2px; + padding: 10px 13px; border-radius: 12px; cursor: pointer; text-align: left; + background: rgba(255,255,255,0.035); border: 1.5px solid rgba(255,255,255,0.08); + transition: all 0.2s ease; position: relative; overflow: hidden; +} +.em-step:hover { background: rgba(255,255,255,0.06); border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.4); transform: translateY(-1px); } +.em-step-glyph { font-size: 16px; } +.em-step-name { font-size: 13px; font-weight: 700; color: #fff; } +.em-step-note { font-size: 10.5px; text-transform: uppercase; letter-spacing: 0.5px; color: rgba(255,255,255,0.4); } +.em-step-arrow { align-self: center; color: rgba(255,255,255,0.25); font-size: 16px; } +/* Pinned = will run first */ +.em-step--pinned { + background: rgba(var(--em-accent-rgb, 99,102,241), 0.16); + border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.6); +} +.em-step--pinned .em-step-note { color: rgb(var(--em-accent-rgb, 129,140,248)); font-weight: 800; } +.em-step--pinned::after { content: '📌'; position: absolute; top: 6px; right: 8px; font-size: 11px; } +/* Current = being processed now */ +.em-step--current .em-step-name::after { content: ''; display: inline-block; width: 7px; height: 7px; margin-left: 6px; border-radius: 50%; background: #4ade80; box-shadow: 0 0 8px #4ade80; animation: em-glow-pulse 1.6s ease-in-out infinite; vertical-align: middle; } +.em-step--current { border-color: rgba(74,222,128,0.45); } +.em-step--current .em-step-note { color: #4ade80; } +/* Done = nothing left */ +.em-step--done .em-step-glyph { opacity: 0.5; } +.em-step--done .em-step-note { color: rgba(74,222,128,0.7); } +@media (prefers-reduced-motion: reduce) { .em-step--current .em-step-name::after { animation: none; } } + +/* Section label as a row (coverage % relocated here from the hero) */ +.em-section-label--row { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; } +.em-coverage-overall { font-size: 12.5px; font-weight: 500; letter-spacing: 0; text-transform: none; color: rgba(255,255,255,0.55); } +.em-coverage-overall strong { color: rgb(var(--em-accent-rgb, 129,140,248)); font-weight: 800; } /* Accent-themed buttons / active states inside the panel */ .em-panel .em-btn { border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.45); background: rgba(var(--em-accent-rgb, 99,102,241), 0.16); } .em-panel .em-btn:hover:not(:disabled) { background: rgba(var(--em-accent-rgb, 99,102,241), 0.3); } From 62ee1f852048e30e6b3bb5ee67cd57b3630d9a38 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 19:50:57 -0700 Subject: [PATCH 026/108] Enrichment manager: 6 UX improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - #1 Unconfigured-source banner: when a source has enabled=false, show a notice that browsing works but matches/retries won't run until it's set up. - #2 Rate-limit detail: when rate_limited, surface 'resumes in ~Xm' (from the status payload) instead of just a pill. - #3 Richer rows: unmatched items now show parent context — an album's artist, a track's album — via a parent expression in the query (+ test). - #4 Bulk select: per-row checkboxes + a bulk bar to retry several at once (capped concurrency), reusing the /retry item endpoint. - #5 Remember last worker: selection persists in localStorage and is restored on open; openEnrichmentManager(workerId) supports future deep-linking (bubbles left on their pause-on-click behaviour). - #6 Keyboard nav: ArrowUp/Down moves focus between rows; actions are native buttons (Enter/Space) and Escape closes — list isn't poll-refreshed so focus is stable. 53 enrichment tests green; JS syntax clean. --- core/enrichment/unmatched.py | 15 +++- tests/enrichment/test_unmatched.py | 10 +++ webui/static/enrichment-manager.js | 135 ++++++++++++++++++++++++++--- webui/static/style.css | 15 ++++ 4 files changed, 161 insertions(+), 14 deletions(-) diff --git a/core/enrichment/unmatched.py b/core/enrichment/unmatched.py index fdf9fbfb..1bcd5eae 100644 --- a/core/enrichment/unmatched.py +++ b/core/enrichment/unmatched.py @@ -34,21 +34,26 @@ SERVICE_ENTITY_SUPPORT = { 'amazon': ('artist', 'album', 'track'), } -# entity_type -> table / display-name column / image expression / optional join. +# entity_type -> table / display-name column / image expression / optional join +# / parent-context expression (the artist an album belongs to; the album a +# track belongs to) so the UI can disambiguate same-named items. # tracks carry no artwork column of their own, so we borrow the parent album's. _ENTITY_TABLE = { 'artist': { 'table': 'artists', 'name': 'name', - 'image': 'artists.thumb_url', 'join': '', + 'image': 'artists.thumb_url', 'join': '', 'parent': None, }, 'album': { 'table': 'albums', 'name': 'title', - 'image': 'albums.thumb_url', 'join': '', + 'image': 'albums.thumb_url', + 'join': 'LEFT JOIN artists par ON albums.artist_id = par.id', + 'parent': 'par.name', }, 'track': { 'table': 'tracks', 'name': 'title', 'image': 'al.thumb_url', 'join': 'LEFT JOIN albums al ON tracks.album_id = al.id', + 'parent': 'al.title', }, } @@ -134,9 +139,11 @@ def build_unmatched_query( where.append(f"{table}.{name_col} LIKE ?") params.append(f"%{query}%") + parent_expr = meta.get('parent') + parent_select = f"{parent_expr} AS parent" if parent_expr else "NULL AS parent" sql = ( f"SELECT {table}.id AS id, {table}.{name_col} AS name, " - f"{image_expr} AS image_url, {table}.{ms} AS status, " + f"{image_expr} AS image_url, {parent_select}, {table}.{ms} AS status, " f"{table}.{la} AS last_attempted " f"FROM {table} {join} " f"WHERE {' AND '.join(where)} " diff --git a/tests/enrichment/test_unmatched.py b/tests/enrichment/test_unmatched.py index d6c88f08..1f749568 100644 --- a/tests/enrichment/test_unmatched.py +++ b/tests/enrichment/test_unmatched.py @@ -150,6 +150,16 @@ def test_track_unmatched_borrows_album_artwork(db): assert res['items'][0]['image_url'] == 'http://img/evolve.jpg' +def test_unmatched_includes_parent_context(db): + # album's parent is its artist; track's parent is its album + album = db.get_enrichment_unmatched('spotify', 'album', status='not_found')['items'][0] + assert album['parent'] == 'Failed Dragons' + track = db.get_enrichment_unmatched('spotify', 'track', status='not_found')['items'][0] + assert track['parent'] == 'Evolve' + artist = db.get_enrichment_unmatched('spotify', 'artist', status='not_found')['items'][0] + assert artist['parent'] is None + + def test_db_raises_on_bad_input(db): with pytest.raises(UnmatchedQueryError): db.get_enrichment_unmatched('spotify', 'artist', status='bogus') diff --git a/webui/static/enrichment-manager.js b/webui/static/enrichment-manager.js index 563111f1..9eadc351 100644 --- a/webui/static/enrichment-manager.js +++ b/webui/static/enrichment-manager.js @@ -83,6 +83,7 @@ const enrichmentManagerState = { page: 0, pageSize: 25, unmatched: null, // { total, items } + selectedItems: new Set(), // ids checked for bulk retry pollTimer: null, loadToken: 0, // guards against out-of-order async renders }; @@ -120,7 +121,7 @@ function _emRelativeTime(value) { // ── Open / close ────────────────────────────────────────────────────────── -async function openEnrichmentManager() { +async function openEnrichmentManager(workerId) { let overlay = document.getElementById('enrichment-manager-overlay'); if (!overlay) { overlay = document.createElement('div'); @@ -157,11 +158,16 @@ async function openEnrichmentManager() { await refreshAllEnrichmentStatuses(); renderEnrichmentRail(); - // Default selection: first running worker, else first in the list. - const running = ENRICHMENT_WORKERS.find( - w => enrichmentManagerState.statuses[w.id]?.running - ); - selectEnrichmentWorker((running || ENRICHMENT_WORKERS[0]).id); + // Selection priority: explicit deep-link arg → last-viewed (remembered) → + // first running worker → first in the list. + let remembered = null; + try { remembered = localStorage.getItem('em-last-worker'); } catch (_e) { /* ignore */ } + const valid = (wid) => wid && _emWorkerById[wid]; + const running = ENRICHMENT_WORKERS.find(w => enrichmentManagerState.statuses[w.id]?.running); + const initial = (valid(workerId) && workerId) + || (valid(remembered) && remembered) + || (running || ENRICHMENT_WORKERS[0]).id; + selectEnrichmentWorker(initial); if (modal) setTimeout(() => modal.focus(), 60); if (enrichmentManagerState.pollTimer) clearInterval(enrichmentManagerState.pollTimer); @@ -307,6 +313,8 @@ function _emUpdateRailRow(id) { async function selectEnrichmentWorker(id) { enrichmentManagerState.selected = id; + try { localStorage.setItem('em-last-worker', id); } catch (_e) { /* ignore */ } + enrichmentManagerState.selectedItems.clear(); enrichmentManagerState.breakdown = null; enrichmentManagerState.unmatched = null; enrichmentManagerState.priority = ''; @@ -389,6 +397,7 @@ function renderEnrichmentPanel() { panel.innerHTML = `
+
Enrichment coverage @@ -397,7 +406,8 @@ function renderEnrichmentPanel() {
-
+ +
`; _emRenderPanelHeader(); @@ -543,6 +553,33 @@ function _emUpdateHeaderLive() { toggle.classList.toggle('em-btn--go', !!isPaused); toggle.textContent = isPaused ? '▶ Resume' : '⏸ Pause'; } + + // #1 unconfigured + #2 rate-limit banner. + const banner = document.getElementById('em-banner'); + if (banner) { + let html = '', cls = 'em-banner'; + if (status && status.enabled === false) { + cls += ' em-banner--warn'; + html = '⚙️ This source isn’t configured — add its credentials in Settings. ' + + 'Browsing works, but matches and retries won’t run until it’s set up.'; + } else if (status && status.rate_limited) { + cls += ' em-banner--warn'; + const rl = status.rate_limit || {}; + const secs = rl.retry_after || rl.reset_in || rl.cooldown_seconds; + const when = secs ? ` — resumes in ~${_emHumanDuration(secs)}` : ' — it will resume automatically'; + html = `⏳ Rate-limited by the source${when}.`; + } + banner.className = cls; + banner.innerHTML = html; + banner.hidden = !html; + } +} + +function _emHumanDuration(seconds) { + const s = Math.max(0, Math.round(Number(seconds) || 0)); + if (s < 60) return `${s}s`; + const m = Math.round(s / 60); + return m < 60 ? `${m}m` : `${Math.round(m / 60)}h`; } function _emRenderStats() { @@ -692,22 +729,96 @@ function _emRenderUnmatchedList() { ? 'not found' : 'pending'; const safeName = _emEscape(item.name || 'Unknown'); + const safeId = _emEscape(item.id); + const parent = item.parent + ? `· ${_emEscape(item.parent)}` + : ''; + const checked = enrichmentManagerState.selectedItems.has(String(item.id)) ? 'checked' : ''; return `
+ ${img}
-
${safeName}
+
${safeName} ${parent}
${statusBadge} ${last}
- + + onclick="retryEnrichmentItem('${id}','${entity}','${safeId}', this)">Retry
`; }).join(''); } _emRenderPager(); + _emRenderBulkBar(); +} + +function _emRenderBulkBar() { + const bar = document.getElementById('em-bulk-bar'); + if (!bar) return; + const n = enrichmentManagerState.selectedItems.size; + if (!n) { bar.hidden = true; bar.innerHTML = ''; return; } + bar.hidden = false; + bar.innerHTML = ` + ${n} selected + + `; +} + +function toggleEnrichmentRowSelect(id, checked) { + if (checked) enrichmentManagerState.selectedItems.add(String(id)); + else enrichmentManagerState.selectedItems.delete(String(id)); + _emRenderBulkBar(); +} + +function clearEnrichmentSelection() { + enrichmentManagerState.selectedItems.clear(); + _emRenderUnmatchedList(); +} + +// #6 keyboard nav: ↑/↓ moves focus between rows' Match buttons (Enter/Space +// then activates natively). The list isn't refreshed by polling, so focus +// stays put between user actions. +function onEnrichmentListKey(e) { + if (e.key !== 'ArrowDown' && e.key !== 'ArrowUp') return; + const host = document.getElementById('em-unmatched-list'); + if (!host) return; + const btns = Array.from(host.querySelectorAll('.em-row .em-row-actions .em-btn:first-child')); + if (!btns.length) return; + e.preventDefault(); + const idx = btns.indexOf(document.activeElement); + const next = e.key === 'ArrowDown' + ? (idx < 0 ? 0 : Math.min(idx + 1, btns.length - 1)) + : (idx <= 0 ? 0 : idx - 1); + btns[next].focus(); +} + +async function retrySelectedEnrichment(btn) { + const service = enrichmentManagerState.selected; + const entity = enrichmentManagerState.entityTab; + const ids = Array.from(enrichmentManagerState.selectedItems); + if (!ids.length) return; + if (btn) { btn.disabled = true; btn.textContent = 'Re-queuing…'; } + let ok = 0; + // Cap concurrency to be gentle on the server. + for (let i = 0; i < ids.length; i += 5) { + const slice = ids.slice(i, i + 5); + const results = await Promise.all(slice.map(eid => + fetch(`/api/enrichment/${service}/retry`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ entity_type: entity, scope: 'item', entity_id: eid }), + }).then(r => r.ok).catch(() => false) + )); + ok += results.filter(Boolean).length; + } + showToast(`Re-queued ${ok.toLocaleString()} item(s)`, ok ? 'success' : 'error'); + enrichmentManagerState.selectedItems.clear(); + await Promise.all([_emLoadBreakdown(service), _emLoadUnmatched()]); + _emRenderStats(); + _emRenderUnmatchedList(); } function _emRenderPager() { @@ -961,4 +1072,8 @@ window.toggleEnrichmentWorker = toggleEnrichmentWorker; window.setEnrichmentPriority = setEnrichmentPriority; window.retryEnrichmentItem = retryEnrichmentItem; window.retryAllFailedEnrichment = retryAllFailedEnrichment; +window.toggleEnrichmentRowSelect = toggleEnrichmentRowSelect; +window.clearEnrichmentSelection = clearEnrichmentSelection; +window.retrySelectedEnrichment = retrySelectedEnrichment; +window.onEnrichmentListKey = onEnrichmentListKey; window.openEnrichmentMatch = openEnrichmentMatch; diff --git a/webui/static/style.css b/webui/static/style.css index 22b2de9a..2caa10e2 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -65170,6 +65170,21 @@ body.em-scroll-lock { overflow: hidden; } .em-search:focus, .em-select:focus { outline: none; border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.6); } .em-row:hover { background: rgba(var(--em-accent-rgb, 255,255,255), 0.07); border-color: rgba(var(--em-accent-rgb, 255,255,255), 0.14); } +/* Disabled / rate-limit banner */ +.em-banner { flex: 0 0 auto; padding: 10px 14px; border-radius: 10px; font-size: 12.5px; line-height: 1.45; + background: rgba(255,255,255,0.05); border: 1px solid rgba(255,255,255,0.1); color: rgba(255,255,255,0.75); } +.em-banner--warn { background: rgba(240,176,80,0.12); border-color: rgba(240,176,80,0.35); color: #f3cd86; } + +/* Row checkbox + parent context (#3 / #4) */ +.em-row-check { width: 16px; height: 16px; flex: 0 0 auto; cursor: pointer; accent-color: rgb(var(--em-accent-rgb, 99,102,241)); } +.em-row-parent { font-size: 12px; font-weight: 500; color: rgba(255,255,255,0.4); } + +/* Bulk action bar */ +.em-bulk-bar { display: flex; align-items: center; gap: 10px; flex: 0 0 auto; padding: 8px 12px; + background: rgba(var(--em-accent-rgb, 99,102,241), 0.12); border: 1px solid rgba(var(--em-accent-rgb, 99,102,241), 0.3); + border-radius: 10px; } +.em-bulk-count { font-size: 13px; font-weight: 700; color: #fff; margin-right: auto; } + /* Bulk retry-all + helper hint */ .em-retry-all { margin-left: 4px; } .em-hint { font-size: 11px; color: rgba(255,255,255,0.38); flex: 0 0 auto; margin-top: -2px; } From f8afe566423da39dff9750a33454bba172b81757 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 19:58:14 -0700 Subject: [PATCH 027/108] Enrichment priority: pinning sweeps the whole group; rail shows live phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligns the 'process this group first' behaviour with intent: - Pinning a group now also re-queues that group's previously-failed (not_found -> pending) items, so the worker processes ALL unmatched in the group (pending + missing), not just never-tried ones. Safe from loops: each is attempted once, still-unmatched return to not_found, and the pending-only worker hook won't re-pick them. Toast reports how many were re-queued. - The left rail now shows each worker's current group while running ('Running · albums'), so you can see what every service is on at a glance. Frontend only; reuses the tested /priority + /retry endpoints. --- webui/static/enrichment-manager.js | 58 +++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/webui/static/enrichment-manager.js b/webui/static/enrichment-manager.js index 9eadc351..82d9a8de 100644 --- a/webui/static/enrichment-manager.js +++ b/webui/static/enrichment-manager.js @@ -264,6 +264,16 @@ function _emOverallPct(status) { return total ? Math.round((matched / total) * 100) : 0; } +// Rail sub-line: while running, show the group it's on ("Running · albums"); +// otherwise the status + overall coverage %. +function _emRailSubText(status) { + const info = _emStatusInfo(status); + const phase = _emCurrentPhase(status); + if (info.cls === 'running' && phase) return `${info.label} · ${_emEntityLabel(phase, true).toLowerCase()}`; + const pct = _emOverallPct(status); + return `${info.label}${pct == null ? '' : ` · ${pct}%`}`; +} + function renderEnrichmentRail() { const rail = document.getElementById('em-rail'); if (!rail) return; @@ -279,7 +289,7 @@ function renderEnrichmentRail() { ${_emIconHtml(w.id)} ${_emEscape(w.name)} - ${info.label}${pct == null ? '' : ` · ${pct}%`} + ${_emRailSubText(status)} ${cov} @@ -304,7 +314,7 @@ function _emUpdateRailRow(id) { const dot = row.querySelector('.em-dot'); if (dot) { dot.className = `em-dot em-dot--${info.cls}`; dot.title = info.label; } const sub = row.querySelector('.em-worker-sub'); - if (sub) sub.textContent = `${info.label}${pct == null ? '' : ` · ${pct}%`}`; + if (sub) sub.textContent = _emRailSubText(status); const cov = row.querySelector('.em-rail-cov-fill'); if (cov && pct != null) cov.style.width = `${pct}%`; } @@ -465,6 +475,7 @@ function _emRenderChain() { async function setEnrichmentPriority(entity) { const id = enrichmentManagerState.selected; + const name = _emWorkerById[id].name; try { const res = await fetch(`/api/enrichment/${id}/priority`, { method: 'POST', @@ -472,15 +483,44 @@ async function setEnrichmentPriority(entity) { body: JSON.stringify({ entity: entity || 'none' }), }); const data = await res.json().catch(() => ({})); - if (res.ok && data.success) { - enrichmentManagerState.priority = data.priority || ''; - showToast(enrichmentManagerState.priority - ? `${_emWorkerById[id].name} will process ${_emEntityLabel(enrichmentManagerState.priority, true).toLowerCase()} first` - : `${_emWorkerById[id].name} back to automatic order`, 'success'); - _emRenderChain(); - } else { + if (!res.ok || !data.success) { showToast(data.error || 'Could not set processing order', 'error'); + return; } + enrichmentManagerState.priority = data.priority || ''; + + if (!enrichmentManagerState.priority) { + showToast(`${name} back to automatic order`, 'success'); + _emRenderChain(); + return; + } + + // Pin means "process the whole group". Re-queue this group's + // previously-failed items (not_found -> pending) so the worker sweeps + // ALL unmatched, not just never-tried ones. Safe: each is attempted + // once; still-unmatched become not_found again and the pending-only + // pin won't re-pick them, so no loop. + let reset = 0; + try { + const rr = await fetch(`/api/enrichment/${id}/retry`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ entity_type: entity, scope: 'failed' }), + }); + const rd = await rr.json().catch(() => ({})); + if (rr.ok) reset = rd.reset || 0; + } catch (_e) { /* priority still set; failed sweep is best-effort */ } + + const label = _emEntityLabel(entity, true).toLowerCase(); + showToast(reset + ? `${name} will process all ${label} first · re-queued ${reset.toLocaleString()} previously-failed` + : `${name} will process ${label} first`, 'success'); + + // Counts changed (failed -> pending) — refresh stats, chain and list. + await Promise.all([_emLoadBreakdown(id), _emLoadUnmatched()]); + _emRenderStats(); + _emRenderChain(); + _emRenderUnmatchedControls(); + _emRenderUnmatchedList(); } catch (_e) { showToast('Error setting processing order', 'error'); } From f827ddc2825ea64292b717620329ecffbee11228 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 20:27:57 -0700 Subject: [PATCH 028/108] Enrichment manager: merge coverage+order into one card set, fix ordering, richer rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses three pieces of UI feedback: - Fix entity order: enrichment coverage was rendering by object-key order (albums first). Now sorted canonically artist → album → track via _emOrderEntities, used everywhere. - Combine 'Processing order' and 'Enrichment coverage' into a single set of entity cards: each card shows coverage (segmented matched/not_found/pending bar + %) AND is the click target to pin that group to enrich first, with live 'Now' / pinned 'First 📌' / 'Done' states and per-worker accent. Drops the two redundant sections (and the old chain/stats renderers). - Richer match rows: status stripe down the left edge (red=not found, amber=pending), larger rounded artwork with a gradient placeholder, parent context (artist/album), and a subtle slide-on-hover. Frontend only; JS syntax clean. --- webui/static/enrichment-manager.js | 202 +++++++++++++---------------- webui/static/style.css | 98 ++++++++------ 2 files changed, 150 insertions(+), 150 deletions(-) diff --git a/webui/static/enrichment-manager.js b/webui/static/enrichment-manager.js index 82d9a8de..f5109c9b 100644 --- a/webui/static/enrichment-manager.js +++ b/webui/static/enrichment-manager.js @@ -94,6 +94,15 @@ function _emEntityLabel(entity, plural) { return plural ? base + 's' : base; } +// Always present entities in the worker's real processing order, regardless of +// the order the API/object keys happen to arrive in. +const _EM_ENTITY_ORDER = { artist: 0, album: 1, track: 2 }; +function _emOrderEntities(list) { + return [...new Set(list || [])].sort( + (a, b) => (_EM_ENTITY_ORDER[a] ?? 9) - (_EM_ENTITY_ORDER[b] ?? 9) + ); +} + function _emEscape(s) { return String(s == null ? '' : s).replace(/[&<>"']/g, c => ( { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c] @@ -207,7 +216,7 @@ async function refreshEnrichmentManager(btn) { renderEnrichmentRail(); const sel = enrichmentManagerState.selected; if (sel) await Promise.all([_emLoadBreakdown(sel), _emLoadUnmatched()]); - _emRenderStats(); + _emRenderEntityCards(); _emRenderUnmatchedList(); _emRenderPanelHeader(); if (btn) setTimeout(() => btn.classList.remove('em-spinning'), 400); @@ -236,7 +245,7 @@ async function _emPollSelected() { enrichmentManagerState.statuses[id] = await res.json(); _emUpdateHeaderLive(); // in-place — no logo reflow/flicker _emUpdateRailRow(id); - _emRenderChain(); // current-phase highlight tracks live + _emRenderEntityCards(); } } catch (_e) { /* transient — keep last */ } } @@ -408,12 +417,11 @@ function renderEnrichmentPanel() { panel.innerHTML = `
-
- Enrichment coverage + Coverage & processing order — click a group to enrich it first
-
+
@@ -421,56 +429,91 @@ function renderEnrichmentPanel() {
`; _emRenderPanelHeader(); - _emRenderChain(); - _emRenderStats(); + _emRenderEntityCards(); _emRenderUnmatchedControls(); _emRenderUnmatchedList(); } -// Processing-order strip: shows the artist→album→track chain, where the worker -// currently is, and lets the user pin one group to run first. -function _emRenderChain() { - const host = document.getElementById('em-chain'); +// Combined coverage + processing-order cards. Each entity card both visualises +// coverage (matched/not_found/pending segmented bar + %) AND is the click target +// to pin that group to enrich first ('now' = live phase, '📌 first' = pinned, +// '✓ done' = nothing left). One section instead of two redundant ones. +function _emRenderEntityCards() { + const host = document.getElementById('em-cards'); if (!host) return; const id = enrichmentManagerState.selected; - const supported = (enrichmentManagerState.unmatched && enrichmentManagerState.unmatched.entity_types) - || (enrichmentManagerState.breakdown && Object.keys(enrichmentManagerState.breakdown)) - || ['artist']; + const bd = enrichmentManagerState.breakdown; + const supported = _emOrderEntities( + (enrichmentManagerState.unmatched && enrichmentManagerState.unmatched.entity_types) + || (bd && Object.keys(bd)) || ['artist'] + ); + + if (!bd) { + host.innerHTML = supported.map(() => ` +
+
+
+
+
`).join(''); + return; + } + const status = enrichmentManagerState.statuses[id]; const phase = _emCurrentPhase(status); const pinned = enrichmentManagerState.priority; - const bd = enrichmentManagerState.breakdown || {}; const glyphs = { artist: '🎤', album: '💿', track: '🎵' }; - const steps = supported.map((e, i) => { + host.innerHTML = supported.map(e => { const d = bd[e] || {}; - const pending = (d.pending || 0) + (d.not_found || 0); - const isCurrent = phase === e; - const isPinned = pinned === e; - const isDone = bd[e] && pending === 0; - const cls = ['em-step', - isPinned ? 'em-step--pinned' : '', - isCurrent ? 'em-step--current' : '', - isDone ? 'em-step--done' : ''].filter(Boolean).join(' '); - const note = isPinned ? 'first' : (isCurrent ? 'now' : (isDone ? 'done' : `${pending.toLocaleString()} left`)); - const arrow = i < supported.length - 1 ? '→' : ''; + const total = d.total || 0, matched = d.matched || 0, nf = d.not_found || 0, pend = d.pending || 0; + const pct = total ? Math.round((matched / total) * 100) : 0; + const seg = (n) => (total ? (n / total) * 100 : 0); + const isPinned = pinned === e, isCurrent = phase === e, isDone = total > 0 && (pend + nf) === 0; + const cls = ['em-card', + isPinned ? 'em-card--pinned' : '', + isCurrent ? 'em-card--current' : '', + isDone ? 'em-card--done' : ''].filter(Boolean).join(' '); + const badge = isPinned + ? '📌 First' + : isCurrent + ? '● Now' + : isDone + ? '✓ Done' + : `${(pend + nf).toLocaleString()} left`; return ` - ${arrow}`; +
+ ${glyphs[e] || '•'} + ${_emEntityLabel(e, true)} + ${badge} + ${pct}% +
+
+
+
+
+
+
+ ${matched.toLocaleString()} + ${nf.toLocaleString()} + ${pend.toLocaleString()} +
+ `; }).join(''); - host.innerHTML = ` -
- Processing order - ${pinned - ? `Pinned ${_emEntityLabel(pinned, true)} first · click again for auto` - : 'Click a group to process it first'} -
-
${steps}
`; + // Overall matched coverage across every entity type. + const overall = document.getElementById('em-coverage-overall'); + if (overall) { + let m = 0, t = 0; + Object.values(bd).forEach(d => { m += d.matched || 0; t += d.total || 0; }); + const pct = t ? Math.round((m / t) * 100) : 0; + overall.innerHTML = t ? `${pct}% matched · ${m.toLocaleString()} of ${t.toLocaleString()}` : ''; + } + + requestAnimationFrame(() => { + host.querySelectorAll('.em-seg-fill').forEach(el => { el.style.width = `${el.dataset.pct || 0}%`; }); + }); } async function setEnrichmentPriority(entity) { @@ -491,7 +534,7 @@ async function setEnrichmentPriority(entity) { if (!enrichmentManagerState.priority) { showToast(`${name} back to automatic order`, 'success'); - _emRenderChain(); + _emRenderEntityCards(); return; } @@ -515,10 +558,9 @@ async function setEnrichmentPriority(entity) { ? `${name} will process all ${label} first · re-queued ${reset.toLocaleString()} previously-failed` : `${name} will process ${label} first`, 'success'); - // Counts changed (failed -> pending) — refresh stats, chain and list. + // Counts changed (failed -> pending) — refresh cards + list. await Promise.all([_emLoadBreakdown(id), _emLoadUnmatched()]); - _emRenderStats(); - _emRenderChain(); + _emRenderEntityCards(); _emRenderUnmatchedControls(); _emRenderUnmatchedList(); } catch (_e) { @@ -622,69 +664,6 @@ function _emHumanDuration(seconds) { return m < 60 ? `${m}m` : `${Math.round(m / 60)}h`; } -function _emRenderStats() { - const host = document.getElementById('em-stats'); - if (!host) return; - const bd = enrichmentManagerState.breakdown; - if (!bd) { - // Skeleton cards (count unknown yet — 3 covers the common case). - host.innerHTML = Array.from({ length: 3 }, () => ` -
-
-
-
-
`).join(''); - return; - } - - const glyphs = { artist: '🎤', album: '💿', track: '🎵' }; - host.innerHTML = Object.keys(bd).map(entity => { - const d = bd[entity] || {}; - const total = d.total || 0; - const matched = d.matched || 0; - const notFound = d.not_found || 0; - const pending = d.pending || 0; - const pct = total ? Math.round((matched / total) * 100) : 0; - const seg = (n) => (total ? (n / total) * 100 : 0); - return ` -
-
- ${glyphs[entity] || '•'}${_emEntityLabel(entity, true)} - ${pct}% -
-
-
-
-
-
-
- ${matched.toLocaleString()} matched - ${notFound.toLocaleString()} missed - ${pending.toLocaleString()} pending -
-
`; - }).join(''); - - // Overall matched coverage across every entity type (relocated here from - // the hero per the refined-banner header). - const overall = document.getElementById('em-coverage-overall'); - if (overall) { - let m = 0, t = 0; - Object.values(bd).forEach(d => { m += d.matched || 0; t += d.total || 0; }); - const pct = t ? Math.round((m / t) * 100) : 0; - overall.innerHTML = t - ? `${pct}% matched · ${m.toLocaleString()} of ${t.toLocaleString()}` - : ''; - } - - // Animate the segments in from 0 on the next frame (CSS transition does the rest). - requestAnimationFrame(() => { - host.querySelectorAll('.em-seg-fill').forEach(el => { - el.style.width = `${el.dataset.pct || 0}%`; - }); - }); -} - function _emRenderUnmatchedControls() { const host = document.getElementById('em-unmatched-controls'); if (!host) return; @@ -774,8 +753,9 @@ function _emRenderUnmatchedList() { ? `· ${_emEscape(item.parent)}` : ''; const checked = enrichmentManagerState.selectedItems.has(String(item.id)) ? 'checked' : ''; + const rowCls = item.status === 'not_found' ? 'em-row em-row--nf' : 'em-row em-row--pend'; return ` -
+
@@ -857,7 +837,7 @@ async function retrySelectedEnrichment(btn) { showToast(`Re-queued ${ok.toLocaleString()} item(s)`, ok ? 'success' : 'error'); enrichmentManagerState.selectedItems.clear(); await Promise.all([_emLoadBreakdown(service), _emLoadUnmatched()]); - _emRenderStats(); + _emRenderEntityCards(); _emRenderUnmatchedList(); } @@ -944,7 +924,7 @@ async function retryEnrichmentItem(service, entityType, entityId, btn) { if (res.ok && data.success) { showToast('Re-queued for enrichment', 'success'); await Promise.all([_emLoadBreakdown(service), _emLoadUnmatched()]); - _emRenderStats(); + _emRenderEntityCards(); _emRenderUnmatchedList(); } else { showToast(data.error || 'Failed to re-queue', 'error'); @@ -976,7 +956,7 @@ async function retryAllFailedEnrichment(btn) { showToast(`Re-queued ${(data.reset || 0).toLocaleString()} item(s)`, 'success'); enrichmentManagerState.page = 0; await Promise.all([_emLoadBreakdown(service), _emLoadUnmatched()]); - _emRenderStats(); + _emRenderEntityCards(); _emRenderUnmatchedControls(); _emRenderUnmatchedList(); } else { @@ -1089,7 +1069,7 @@ async function _emApplyMatch(entityType, entityId, service, serviceId, overlay) // Refresh the manager's stats + list for the *selected* worker. const sel = enrichmentManagerState.selected; await Promise.all([_emLoadBreakdown(sel), _emLoadUnmatched()]); - _emRenderStats(); + _emRenderEntityCards(); _emRenderUnmatchedList(); } else { showToast(data.error || 'Failed to match', 'error'); diff --git a/webui/static/style.css b/webui/static/style.css index 2caa10e2..db31b18a 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -64978,16 +64978,34 @@ body.reduce-effects .dash-card::after { .em-unmatched-list { flex: 1 1 auto; overflow-y: auto; display: flex; flex-direction: column; gap: 6px; padding-right: 4px; } .em-row { - display: flex; align-items: center; gap: 12px; padding: 9px 12px; - background: rgba(255,255,255,0.03); border: 1px solid rgba(255,255,255,0.06); border-radius: 10px; + display: flex; align-items: center; gap: 13px; padding: 10px 14px 10px 16px; + background: linear-gradient(100deg, rgba(255,255,255,0.045), rgba(255,255,255,0.02)); + border: 1px solid rgba(255,255,255,0.07); border-radius: 12px; + position: relative; overflow: hidden; + transition: transform 0.15s ease, border-color 0.15s ease, background 0.15s ease; } -.em-row:hover { background: rgba(255,255,255,0.06); } -.em-row-img { width: 42px; height: 42px; border-radius: 8px; object-fit: cover; flex: 0 0 auto; } -.em-row-img--ph { display: flex; align-items: center; justify-content: center; background: rgba(255,255,255,0.08); color: rgba(255,255,255,0.4); font-size: 18px; } +/* status stripe down the left edge */ +.em-row::before { + content: ''; position: absolute; left: 0; top: 0; bottom: 0; width: 3px; + background: rgba(255,255,255,0.15); +} +.em-row--nf::before { background: #e0586b; } +.em-row--pend::before { background: #f0c060; } +.em-row:hover { + transform: translateX(2px); + background: linear-gradient(100deg, rgba(255,255,255,0.07), rgba(255,255,255,0.03)); + border-color: rgba(var(--em-accent-rgb, 255,255,255), 0.22); +} +.em-row-img { width: 48px; height: 48px; border-radius: 10px; object-fit: cover; flex: 0 0 auto; + box-shadow: 0 2px 8px rgba(0,0,0,0.3); } +.em-row-img--ph { display: flex; align-items: center; justify-content: center; + background: linear-gradient(135deg, rgba(var(--em-accent-rgb, 99,102,241), 0.3), rgba(255,255,255,0.06)); + color: rgba(255,255,255,0.55); font-size: 20px; } .em-row-info { flex: 1 1 auto; min-width: 0; } -.em-row-name { font-size: 14px; font-weight: 600; color: #fff; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } -.em-row-meta { display: flex; gap: 8px; align-items: center; margin-top: 3px; font-size: 11px; } -.em-row-actions { display: flex; gap: 6px; flex: 0 0 auto; } +.em-row-name { font-size: 14.5px; font-weight: 600; color: #fff; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +.em-row-meta { display: flex; gap: 8px; align-items: center; margin-top: 4px; font-size: 11px; } +.em-row-actions { display: flex; gap: 6px; flex: 0 0 auto; opacity: 0.85; transition: opacity 0.15s ease; } +.em-row:hover .em-row-actions { opacity: 1; } .em-empty { text-align: center; padding: 48px 20px; color: rgba(255,255,255,0.55); font-size: 15px; } .em-empty-emoji { font-size: 38px; margin-bottom: 10px; opacity: 0.85; } .em-pager { display: flex; align-items: center; justify-content: center; gap: 14px; flex: 0 0 auto; padding-top: 4px; font-size: 12px; } @@ -65093,39 +65111,41 @@ body.em-scroll-lock { overflow: hidden; } .em-ph-name-sub { font-size: 14px; font-weight: 500; color: rgba(255,255,255,0.45); } .em-hero .em-ph-actions { z-index: 1; } -/* Processing-order chain strip */ -.em-chain { display: flex; flex-direction: column; gap: 9px; flex: 0 0 auto; } -.em-chain-head { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; flex-wrap: wrap; } -.em-chain-hint { font-size: 11.5px; color: rgba(255,255,255,0.4); } -.em-chain-hint strong { color: rgb(var(--em-accent-rgb, 129,140,248)); } -.em-chain-flow { display: flex; align-items: stretch; gap: 8px; flex-wrap: wrap; } -.em-step { - flex: 1 1 0; min-width: 120px; - display: flex; flex-direction: column; align-items: flex-start; gap: 2px; - padding: 10px 13px; border-radius: 12px; cursor: pointer; text-align: left; - background: rgba(255,255,255,0.035); border: 1.5px solid rgba(255,255,255,0.08); - transition: all 0.2s ease; position: relative; overflow: hidden; +/* Combined coverage + processing-order cards (click to enrich a group first) */ +.em-section-sub { font-weight: 500; text-transform: none; letter-spacing: 0; color: rgba(255,255,255,0.35); font-size: 11px; } +.em-cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(215px, 1fr)); gap: 12px; flex: 0 0 auto; } +.em-card { + display: flex; flex-direction: column; gap: 10px; text-align: left; cursor: pointer; + padding: 14px; border-radius: 14px; position: relative; overflow: hidden; + background: linear-gradient(160deg, rgba(255,255,255,0.06), rgba(255,255,255,0.02)); + border: 1.5px solid rgba(255,255,255,0.08); + transition: transform 0.2s ease, border-color 0.2s ease, background 0.2s ease; } -.em-step:hover { background: rgba(255,255,255,0.06); border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.4); transform: translateY(-1px); } -.em-step-glyph { font-size: 16px; } -.em-step-name { font-size: 13px; font-weight: 700; color: #fff; } -.em-step-note { font-size: 10.5px; text-transform: uppercase; letter-spacing: 0.5px; color: rgba(255,255,255,0.4); } -.em-step-arrow { align-self: center; color: rgba(255,255,255,0.25); font-size: 16px; } -/* Pinned = will run first */ -.em-step--pinned { - background: rgba(var(--em-accent-rgb, 99,102,241), 0.16); - border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.6); +.em-card:hover { + transform: translateY(-2px); + border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.45); + background: linear-gradient(160deg, rgba(255,255,255,0.09), rgba(255,255,255,0.03)); } -.em-step--pinned .em-step-note { color: rgb(var(--em-accent-rgb, 129,140,248)); font-weight: 800; } -.em-step--pinned::after { content: '📌'; position: absolute; top: 6px; right: 8px; font-size: 11px; } -/* Current = being processed now */ -.em-step--current .em-step-name::after { content: ''; display: inline-block; width: 7px; height: 7px; margin-left: 6px; border-radius: 50%; background: #4ade80; box-shadow: 0 0 8px #4ade80; animation: em-glow-pulse 1.6s ease-in-out infinite; vertical-align: middle; } -.em-step--current { border-color: rgba(74,222,128,0.45); } -.em-step--current .em-step-note { color: #4ade80; } -/* Done = nothing left */ -.em-step--done .em-step-glyph { opacity: 0.5; } -.em-step--done .em-step-note { color: rgba(74,222,128,0.7); } -@media (prefers-reduced-motion: reduce) { .em-step--current .em-step-name::after { animation: none; } } +.em-card-top { display: flex; align-items: center; gap: 8px; } +.em-card-glyph { font-size: 15px; } +.em-card-title { font-size: 12.5px; font-weight: 800; color: #fff; text-transform: uppercase; letter-spacing: 0.5px; } +.em-card-pct { margin-left: auto; font-size: 21px; font-weight: 800; color: #fff; line-height: 1; } +.em-card-badge { font-size: 9.5px; font-weight: 800; text-transform: uppercase; letter-spacing: 0.5px; padding: 2px 7px; border-radius: 999px; background: rgba(255,255,255,0.1); color: rgba(255,255,255,0.55); white-space: nowrap; } +.em-card-badge--pin { background: rgba(var(--em-accent-rgb, 99,102,241), 0.28); color: #fff; } +.em-card-badge--now { background: rgba(74,222,128,0.2); color: #4ade80; } +.em-card-badge--done { background: rgba(74,222,128,0.14); color: rgba(74,222,128,0.85); } +.em-card .em-stat-legend { gap: 12px; margin-top: 0; } +.em-card .em-leg { font-size: 11.5px; color: rgba(255,255,255,0.7); } +/* States */ +.em-card--pinned { + border-color: rgba(var(--em-accent-rgb, 99,102,241), 0.65); + background: linear-gradient(160deg, rgba(var(--em-accent-rgb, 99,102,241), 0.16), rgba(255,255,255,0.02)); + box-shadow: 0 0 0 1px rgba(var(--em-accent-rgb, 99,102,241), 0.25), 0 8px 24px rgba(var(--em-accent-rgb, 99,102,241), 0.15); +} +.em-card--current { border-color: rgba(74,222,128,0.4); } +.em-card--current .em-card-badge--now { animation: em-glow-pulse 1.8s ease-in-out infinite; } +.em-card--done { opacity: 0.74; } +@media (prefers-reduced-motion: reduce) { .em-card--current .em-card-badge--now { animation: none; } } /* Section label as a row (coverage % relocated here from the hero) */ .em-section-label--row { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; } From 3a560bd1bb9c38167363a8de5767e47455b17dcc Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Tue, 2 Jun 2026 20:41:05 -0700 Subject: [PATCH 029/108] Enrichment manager: rebuilt header + global process-first + cleaner rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rebuilt the modal header: gradient top bar with a glowing 🧬 icon chip, gradient title + subtitle, and styled refresh/close — replaces the flat bar. - Global 'process first everywhere' control in the header: Artists/Albums/ Tracks/Auto applies to every worker at once (workers that don't enrich a group are skipped via the 400 the endpoint already returns). Sets order only. - Match rows: replaced the loud accent-gradient artwork placeholder with a subtle neutral chip showing the entity glyph; real images layer over it and remove themselves on error, so missing/broken art never leaves ragged gaps. - Removed overflow:hidden from .em-row. Frontend only; JS syntax clean. --- webui/static/enrichment-manager.js | 57 ++++++++++++++++++++++++++---- webui/static/style.css | 56 +++++++++++++++++++++++------ 2 files changed, 96 insertions(+), 17 deletions(-) diff --git a/webui/static/enrichment-manager.js b/webui/static/enrichment-manager.js index f5109c9b..f31e46f9 100644 --- a/webui/static/enrichment-manager.js +++ b/webui/static/enrichment-manager.js @@ -140,12 +140,26 @@ async function openEnrichmentManager(workerId) { overlay.innerHTML = ` - +
`; document.body.appendChild(overlay); const input = overlay.querySelector('#artmap-explore-input'); - const goBtn = overlay.querySelector('#artmap-explore-go'); + const results = overlay.querySelector('#artmap-explore-results'); + const spinner = overlay.querySelector('#artmap-explore-spinner'); + overlay.querySelector('#artmap-explore-cancel').onclick = () => finish(null); - const submit = () => { - const val = input.value.trim(); - overlay.remove(); - resolve(val || null); + const renderResults = (artists) => { + results.innerHTML = ''; + if (!artists.length) { + results.innerHTML = '
No artists found
'; + return; + } + artists.forEach(a => { + const img = a.image_url || '/static/placeholder-album.png'; + const row = document.createElement('div'); + row.className = 'artmap-explore-result'; + row.innerHTML = ` + + ${escapeHtml(a.name)} + Explore →`; + row.onclick = () => finish(a.name); // pick the resolved artist, not raw text + results.appendChild(row); + }); }; - goBtn.onclick = submit; - input.addEventListener('keydown', (e) => { if (e.key === 'Enter') submit(); }); + let timer = null; + let token = 0; + const doSearch = () => { + const q = input.value.trim(); + if (!q) { results.innerHTML = ''; spinner.style.display = 'none'; clearTimeout(timer); return; } + clearTimeout(timer); + timer = setTimeout(async () => { + const myToken = ++token; + spinner.style.display = 'flex'; + try { + const resp = await fetch(`/api/discover/build-playlist/search-artists?query=${encodeURIComponent(q)}`); + const data = await resp.json(); + if (myToken !== token) return; // a newer keystroke superseded this + renderResults((data && data.success && Array.isArray(data.artists)) ? data.artists : []); + } catch (e) { + if (myToken === token) results.innerHTML = '
Search failed — try again
'; + } finally { + if (myToken === token) spinner.style.display = 'none'; + } + }, 350); + }; + + input.addEventListener('input', doSearch); + input.addEventListener('keydown', (e) => { + if (e.key === 'Enter') { + const first = results.querySelector('.artmap-explore-result'); + if (first) first.click(); // Enter = pick top match, never raw text + } else if (e.key === 'Escape') { + finish(null); + } + }); setTimeout(() => input.focus(), 50); }); } diff --git a/webui/static/style.css b/webui/static/style.css index f5b13454..35afbdef 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -34358,6 +34358,36 @@ div.artist-hero-badge { .artmap-explore-input::placeholder { color: rgba(255,255,255,0.2); } .artmap-search-prompt-actions { display: flex; justify-content: flex-end; gap: 8px; } +/* Artist Explorer — search + select picker */ +.artmap-explore-search-wrap { position: relative; } +.artmap-explore-search-wrap .artmap-explore-input { margin-bottom: 12px; } +.artmap-explore-spinner { position: absolute; right: 12px; top: 11px; pointer-events: none; } +.artmap-explore-spinner .watch-all-loading-spinner { width: 18px; height: 18px; border-width: 2px; } +.artmap-explore-results { + max-height: 320px; overflow-y: auto; display: flex; flex-direction: column; gap: 4px; + margin-bottom: 18px; +} +.artmap-explore-results:empty { margin-bottom: 0; } +.artmap-explore-result { + display: flex; align-items: center; gap: 12px; padding: 8px 10px; border-radius: 10px; + cursor: pointer; transition: background 0.15s ease; +} +.artmap-explore-result:hover { background: rgba(138,43,226,0.14); } +.artmap-explore-result img { + width: 40px; height: 40px; border-radius: 50%; object-fit: cover; flex: 0 0 auto; + background: rgba(255,255,255,0.06); +} +.artmap-explore-result-name { + flex: 1 1 auto; min-width: 0; color: #fff; font-size: 14px; font-weight: 600; + white-space: nowrap; overflow: hidden; text-overflow: ellipsis; +} +.artmap-explore-result-go { + flex: 0 0 auto; font-size: 11px; font-weight: 700; color: rgba(138,43,226,0.9); + opacity: 0; transition: opacity 0.15s ease; +} +.artmap-explore-result:hover .artmap-explore-result-go { opacity: 1; } +.artmap-explore-empty { padding: 18px; text-align: center; color: rgba(255,255,255,0.35); font-size: 13px; } + /* Genre picker modal */ .artmap-genre-picker-modal { background: linear-gradient(165deg, #1e1e32 0%, #181828 100%); From 77d6d49069d67ff8d32e4275598e7a2e73c9d500 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 07:34:41 -0700 Subject: [PATCH 041/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20fix=20r?= =?UTF-8?q?egression=20+=20add=20perf=20overlay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - REVERT the spatial-grid hit-test I added in Phase 1. It inserted each node into every grid cell its bounding box overlaps; the genre map's huge cluster nodes span an enormous number of cells, so the first hover/click triggered a multi-second synchronous build → 'can't hover or click' freeze. Back to a flat O(N) single-pass hit-test (no per-move sort) — sub-ms even for thousands of nodes, can't lock up. - Keep the safe Phase 1 wins (render coalescing, tooltip de-churn, solid-stroke connection lines). - Add a perf overlay toggled with 'd' on the map: shows node/edge counts, the offscreen buffer size + scale, zoom, and the last buffer-rebuild + draw times. So we can measure the real drag/zoom bottleneck (buffer rebuild) instead of optimising blind. JS clean; 64 integrity tests pass. --- webui/static/discover.js | 79 +++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index bd378cc7..6e3230b3 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5515,6 +5515,7 @@ function _artMapRender() { function _artMapDraw() { /**Blit offscreen buffer to screen canvas with pan/zoom. Near-zero cost.**/ + const _t0 = _artMap._perf ? performance.now() : 0; const ctx = _artMap.ctx; const w = _artMap.width; const h = _artMap.height; @@ -5522,8 +5523,12 @@ function _artMapDraw() { ctx.fillStyle = '#0a0a14'; ctx.fillRect(0, 0, w, h); - if (_artMap.dirty || !_artMap.offscreen) _artMapRebuildBuffer(); - if (!_artMap.offscreen) return; + if (_artMap.dirty || !_artMap.offscreen) { + const _rt = _artMap._perf ? performance.now() : 0; + _artMapRebuildBuffer(); + if (_artMap._perf) _artMap._rebuildMs = performance.now() - _rt; + } + if (!_artMap.offscreen) { if (_artMap._perf) _artMapDrawPerf(ctx, _t0); return; } const oc = _artMap.offscreen; const s = _artMap._bufferScale; @@ -5702,6 +5707,37 @@ function _artMapDraw() { _artMap._ripple = null; } } + + if (_artMap._perf) _artMapDrawPerf(ctx, _t0); +} + +// Toggle with 'd' on the map. Shows where frame time goes so we optimise the +// real bottleneck (buffer rebuild on zoom vs. blit on pan) instead of guessing. +function _artMapDrawPerf(ctx, t0) { + const drawMs = performance.now() - t0; + const now = performance.now(); + const dt = _artMap._lastPerfTs ? now - _artMap._lastPerfTs : 0; + _artMap._lastPerfTs = now; + const fps = dt > 0 ? Math.round(1000 / dt) : 0; + const oc = _artMap.offscreen; + const lines = [ + `nodes ${_artMap.placed.length} edges ${(_artMap.edges || []).length}`, + `buffer ${oc ? oc.width + '×' + oc.height : '—'} scale ${(_artMap._bufferScale || 0).toFixed(3)}`, + `zoom ${_artMap.zoom.toFixed(3)}`, + `rebuild ${(_artMap._rebuildMs || 0).toFixed(1)}ms draw ${drawMs.toFixed(1)}ms`, + `~${fps} fps (while interacting)`, + ]; + ctx.save(); + ctx.setTransform(1, 0, 0, 1, 0, 0); // device pixels, ignore dpr scale + ctx.font = '12px monospace'; + ctx.textAlign = 'left'; + ctx.textBaseline = 'top'; + const pad = 8, lh = 16; + ctx.fillStyle = 'rgba(0,0,0,0.72)'; + ctx.fillRect(10, 10, 270, lines.length * lh + pad * 2); + ctx.fillStyle = '#7CFC00'; + lines.forEach((l, i) => ctx.fillText(l, 10 + pad, 10 + pad + i * lh)); + ctx.restore(); } function artMapSearch(query) { @@ -6597,6 +6633,7 @@ function _artMapSetupInteraction(canvas) { else if (e.key === '-') { artMapZoom(0.7); e.preventDefault(); } else if (e.key === '0') { artMapFitToView(); e.preventDefault(); } else if (e.key === 'f' || e.key === 'F') { artMapFitToView(); e.preventDefault(); } + else if (e.key === 'd' || e.key === 'D') { _artMap._perf = !_artMap._perf; _artMapRender(); e.preventDefault(); } else if (e.key === 's' || e.key === 'S') { const input = document.getElementById('artist-map-search'); if (input) { input.focus(); e.preventDefault(); } @@ -6804,40 +6841,14 @@ function _artMapScreenToWorld(e, canvas) { }; } -function _artMapBuildHitGrid() { - // v2 perf: bucket nodes into a coarse world-space grid so hit-testing checks - // only the cell under the cursor instead of sorting+scanning every node each - // mousemove. A node is inserted into every cell its bounding box overlaps, - // so a single-cell lookup at the cursor is exhaustive. - const placed = _artMap.placed || []; - const cell = 400; - const grid = new Map(); - for (const n of placed) { - const mincx = Math.floor((n.x - n.radius) / cell); - const maxcx = Math.floor((n.x + n.radius) / cell); - const mincy = Math.floor((n.y - n.radius) / cell); - const maxcy = Math.floor((n.y + n.radius) / cell); - for (let cx = mincx; cx <= maxcx; cx++) { - for (let cy = mincy; cy <= maxcy; cy++) { - const k = cx + ',' + cy; - let arr = grid.get(k); - if (!arr) { arr = []; grid.set(k, arr); } - arr.push(n); - } - } - } - _artMap._grid = { cell, grid }; - _artMap._gridFor = placed; // identity guard — loaders assign a fresh array -} - function _artMapHitTest(wx, wy) { - if (!_artMap._grid || _artMap._gridFor !== _artMap.placed) _artMapBuildHitGrid(); - const { cell, grid } = _artMap._grid; - const candidates = grid.get(Math.floor(wx / cell) + ',' + Math.floor(wy / cell)); - if (!candidates) return null; - // Watchlist nodes draw on top, so they win ties. + // Single O(N) pass, no per-move sort, no allocation. Watchlist nodes draw on + // top so they win ties; otherwise the first node whose circle contains the + // point wins. (A spatial grid was tried and reverted — it exploded building + // cells for large-radius genre cluster nodes. A flat scan of even thousands + // of nodes is sub-millisecond and can't lock up.) let similarHit = null; - for (const n of candidates) { + for (const n of _artMap.placed) { if ((n.opacity || 0) < 0.3) continue; const dx = wx - n.x, dy = wy - n.y; if (dx * dx + dy * dy <= n.radius * n.radius) { From 884c3b5c07d039c87f37d08abbcd2c1ec604ef11 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 07:54:32 -0700 Subject: [PATCH 042/108] Artist Map perf overlay: also POST timings to app.log (readable server-side) The on-canvas overlay text can't be copied (and can't be grabbed mid-freeze), so when perf mode is on ('d'), the frontend now also POSTs the render timings to /api/discover/artist-map/perf ~1.5x/sec, which logs them as [ARTMAP-PERF] in app.log. Lets the bottleneck be diagnosed from the server side with no manual copying. --- web_server.py | 13 +++++++++++++ webui/static/discover.js | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/web_server.py b/web_server.py index 0e8c0d68..13eb3368 100644 --- a/web_server.py +++ b/web_server.py @@ -28284,6 +28284,19 @@ def get_artist_map_explore(): return _artists_map_get_artist_map_explore() +@app.route('/api/discover/artist-map/perf', methods=['POST']) +def log_artist_map_perf(): + """Debug sink: the artist-map frontend POSTs its render timings here (toggled + with 'd' on the map) so they land in app.log — the on-canvas overlay text + can't be copied. Used to find the real drag/zoom bottleneck.""" + try: + data = request.get_json(silent=True) or {} + logger.info("[ARTMAP-PERF] %s", json.dumps(data, ensure_ascii=False)) + except Exception: + pass + return ('', 204) + + @app.route('/api/discover/build-playlist/search-artists', methods=['GET']) def search_artists_for_playlist(): """Search for artists to use as seeds for custom playlist building""" diff --git a/webui/static/discover.js b/webui/static/discover.js index 6e3230b3..ab244807 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5720,6 +5720,26 @@ function _artMapDrawPerf(ctx, t0) { _artMap._lastPerfTs = now; const fps = dt > 0 ? Math.round(1000 / dt) : 0; const oc = _artMap.offscreen; + + // Ship the numbers to app.log (~1.5/s) so they can be read server-side — + // the on-canvas text below can't be copied, especially mid-lag. + if (!_artMap._perfPostTs || now - _artMap._perfPostTs > 700) { + _artMap._perfPostTs = now; + try { + fetch('/api/discover/artist-map/perf', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + nodes: _artMap.placed.length, edges: (_artMap.edges || []).length, + buffer: oc ? oc.width + 'x' + oc.height : '-', + scale: +(_artMap._bufferScale || 0).toFixed(3), + zoom: +_artMap.zoom.toFixed(3), + rebuildMs: +(_artMap._rebuildMs || 0).toFixed(1), + drawMs: +drawMs.toFixed(1), fps, + }), + }).catch(() => { }); + } catch (e) { /* ignore */ } + } + const lines = [ `nodes ${_artMap.placed.length} edges ${(_artMap.edges || []).length}`, `buffer ${oc ? oc.width + '×' + oc.height : '—'} scale ${(_artMap._bufferScale || 0).toFixed(3)}`, From 6fa733dc6e4e2ab8f159e157aeb47f73a0f5d5e6 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 08:03:30 -0700 Subject: [PATCH 043/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20fix=20t?= =?UTF-8?q?he=20real=20bottleneck:=20cap=20the=20offscreen=20buffer=20(76M?= =?UTF-8?q?P=20=E2=86=92=20~12MP)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Perf telemetry from the genre map (2004 nodes) proved it: the offscreen buffer was 7465×10240 (76 megapixels) — rebuilt in ~979ms on every zoom and blitted at ~150ms/frame (3 fps), with the constellation overlay piling on top. The buffer renders the WHOLE world, and the size cap was 10240px. Cap the max buffer dimension to 4096 (MAX_BUFFER_PX). On the dense genre map that's ~12MP instead of 76MP → ~6x faster rebuild and blit, and more nodes drop under the LOD dot threshold so the rebuild also draws fewer image-clips. The cap only binds on large worlds; small watchlist/explorer maps don't reach it and stay full-resolution. Tunable; perf overlay ('d' → app.log) stays so we can confirm the new numbers. --- webui/static/discover.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index ab244807..4135dbfc 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5004,6 +5004,13 @@ const _artMap = { dirty: true, // true = need to rebuild offscreen buffer WATCHLIST_R: 320, BUFFER: 8, + // Max offscreen-buffer dimension (px). The buffer renders the whole world, + // so on big/dense maps (e.g. 2000-node genre map) this was 10240 → a 76 MP + // canvas that took ~1s to rebuild and ~150ms to blit per frame (3 fps). + // Capping it far lower makes rebuild + blit cheap (and pushes more nodes + // under the LOD dot threshold). Only binds on large worlds — small maps + // stay crisp via the z*2 / 1.0 caps. Tunable. + MAX_BUFFER_PX: 4096, }; async function openArtistMap() { @@ -5348,7 +5355,7 @@ function _artMapRebuildBuffer() { const bh = maxY - minY; // Scale based on zoom — higher zoom = higher res buffer, capped for memory const z = _artMap.zoom || 0.1; - const scale = Math.min(z * 2, 1.0, 10240 / Math.max(bw, bh)); + const scale = Math.min(z * 2, 1.0, _artMap.MAX_BUFFER_PX / Math.max(bw, bh)); if (!_artMap.offscreen) _artMap.offscreen = document.createElement('canvas'); const oc = _artMap.offscreen; From 20b77a774ac04d0c38889e139c531f9e98e70643 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 08:18:41 -0700 Subject: [PATCH 044/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20the=20r?= =?UTF-8?q?eal=20lock:=20downscale=20artist=20images=20on=20decode=20(6GB?= =?UTF-8?q?=20=E2=86=92=20~100MB)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Perf telemetry was the giveaway: after the buffer cap, rebuild + draw were both ~10ms, yet fps stayed 1-3 and the browser 'locked'. Cheap draw + locked system = memory/GPU thrash, not drawing. Cause: artist images load at up to 1000×1000, and a dense map holds ~1500 of them — ~1500 × 1000² × 4B ≈ 6 GB of decoded ImageBitmap memory. The browser GCs/ evicts textures constantly → systemic lag the canvas timers don't see. Fix: decode straight to a 128px avatar via createImageBitmap resize options (nodes render tiny anyway). ~1500 × 128² × 4B ≈ 100 MB instead of 6 GB. Falls back to full decode on engines that ignore the resize opts. This is the one that should actually make it smooth. Perf overlay stays on 'd'. --- webui/static/discover.js | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index 4135dbfc..f638acdf 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -6606,16 +6606,33 @@ function artMapToggleSimilar() { showToast(_artMap._hideSimilar ? 'Showing watchlist only' : 'Showing all artists', 'info', 1500); } +// Artist images come in at up to 1000×1000. Nodes are drawn tiny, so holding +// full-res bitmaps is pointless and ruinous: ~1500 nodes × 1000² × 4 bytes ≈ 6 GB +// of decoded image memory → GC/GPU thrash that locks the browser even though the +// per-frame draw is cheap. Decode straight to a small avatar (~128px) so the +// whole map's images fit in ~100 MB instead of gigabytes. +const _ARTMAP_IMG_PX = 128; +function _artMapDecodeSmall(blob) { + if (!blob) return Promise.resolve(null); + try { + return createImageBitmap(blob, { + resizeWidth: _ARTMAP_IMG_PX, resizeHeight: _ARTMAP_IMG_PX, resizeQuality: 'medium', + }).catch(() => createImageBitmap(blob).catch(() => null)); // older engines ignore opts + } catch (e) { + return createImageBitmap(blob).catch(() => null); + } +} + function _artMapLoadImage(url) { // Try direct CORS fetch first (zero server load, works for Spotify/iTunes/Discogs) return fetch(url, { mode: 'cors' }) .then(r => r.ok ? r.blob() : Promise.reject('not ok')) - .then(b => createImageBitmap(b)) + .then(b => _artMapDecodeSmall(b)) .catch(() => { // Fallback: server proxy for CDNs without CORS headers return fetch('/api/image-proxy?url=' + encodeURIComponent(url)) .then(r => r.ok ? r.blob() : null) - .then(b => b ? createImageBitmap(b) : null) + .then(b => _artMapDecodeSmall(b)) .catch(() => null); }); } From c97108ff6bf247f7e6164e1a6df91bf8bd715dfe Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 08:25:34 -0700 Subject: [PATCH 045/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20Phase?= =?UTF-8?q?=203=20polish:=20crisp=20adaptive=20images,=20snappier=20glowin?= =?UTF-8?q?g=20hover,=20premium=20backdrop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Images: decode adaptively (focal/watchlist nodes ~256-384px, small nodes ~112-150px) instead of a flat 128px — crisp where it matters, memory still bounded (~150-250MB, not 6GB). Fixes the low-res look. - Hover constellation: drop the activation delay 800ms → 220ms (it felt 'gone' because nothing happened for nearly a second), and draw the connection lines as a wide-faint halo + crisp core (a real glow) with no per-frame gradients or shadowBlur — stays cheap. - Backdrop: subtle cached radial glow + vignette behind the map for depth instead of a flat fill (one cheap fillRect/frame). JS clean; 64 integrity tests pass. --- webui/static/discover.js | 64 ++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index f638acdf..c40f30b9 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5231,7 +5231,7 @@ async function openArtistMap() { const n = imgNodes[idx++]; if (_artMap.images[n.id]) { loaded++; continue; } batch.push( - _artMapLoadImage(n.image_url) + _artMapLoadImage(n.image_url, _artMapNodeImgPx(n)) .then(bmp => { if (bmp) _artMap.images[n.id] = bmp; }) .finally(() => { loaded++; @@ -5530,6 +5530,19 @@ function _artMapDraw() { ctx.fillStyle = '#0a0a14'; ctx.fillRect(0, 0, w, h); + // Premium backdrop: a soft central glow fading to a dark vignette. The + // gradient is cached and only rebuilt on resize, so it's one cheap fillRect. + if (!_artMap._bgGrad || _artMap._bgW !== w || _artMap._bgH !== h) { + const g = ctx.createRadialGradient(w / 2, h * 0.42, Math.min(w, h) * 0.12, + w / 2, h / 2, Math.max(w, h) * 0.78); + g.addColorStop(0, 'rgba(46,34,78,0.40)'); + g.addColorStop(0.5, 'rgba(16,12,28,0.0)'); + g.addColorStop(1, 'rgba(0,0,0,0.55)'); + _artMap._bgGrad = g; _artMap._bgW = w; _artMap._bgH = h; + } + ctx.fillStyle = _artMap._bgGrad; + ctx.fillRect(0, 0, w, h); + if (_artMap.dirty || !_artMap.offscreen) { const _rt = _artMap._perf ? performance.now() : 0; _artMapRebuildBuffer(); @@ -5603,17 +5616,21 @@ function _artMapDraw() { ctx.globalAlpha = 1; ctx.restore(); - // Draw connection lines — ONE path, ONE solid stroke. The old - // code built a fresh linear-gradient object per line every frame, - // which is the main cause of the hover-lag on dense maps. - ctx.strokeStyle = `rgba(138,43,226,${0.42 * cFade})`; - ctx.lineWidth = 2; + // Connection lines — build the path ONCE, then two cheap strokes + // (wide faint halo + crisp core) for a glow look without per-frame + // gradients or shadowBlur (those were the hover-lag culprits). + ctx.lineCap = 'round'; ctx.beginPath(); for (const cn of highlightNodes) { if (cn === n) continue; ctx.moveTo(n.x, n.y); ctx.lineTo(cn.x, cn.y); } + ctx.strokeStyle = `rgba(168,85,247,${0.18 * cFade})`; + ctx.lineWidth = 6; + ctx.stroke(); + ctx.strokeStyle = `rgba(201,150,255,${0.6 * cFade})`; + ctx.lineWidth = 1.5; ctx.stroke(); // Redraw highlighted nodes on top @@ -6242,7 +6259,7 @@ async function _openGenreMapWithSelection(selectedGenre) { const batch = []; while (idx < imgNodes.length && batch.length < CONCURRENT) { const n = imgNodes[idx++]; - batch.push(_artMapLoadImage(n.image_url) + batch.push(_artMapLoadImage(n.image_url, _artMapNodeImgPx(n)) .then(bmp => { if (bmp) _artMap.images[n.id] = bmp; }) .finally(() => { loaded++; })); } @@ -6469,7 +6486,7 @@ async function _openArtistMapExplorerWithName(name) { const batch = []; while (idx < imgNodes.length && batch.length < CONCURRENT) { const n = imgNodes[idx++]; - batch.push(_artMapLoadImage(n.image_url) + batch.push(_artMapLoadImage(n.image_url, _artMapNodeImgPx(n)) .then(bmp => { if (bmp) _artMap.images[n.id] = bmp; }) .finally(() => { loaded++; })); } @@ -6611,32 +6628,46 @@ function artMapToggleSimilar() { // of decoded image memory → GC/GPU thrash that locks the browser even though the // per-frame draw is cheap. Decode straight to a small avatar (~128px) so the // whole map's images fit in ~100 MB instead of gigabytes. -const _ARTMAP_IMG_PX = 128; -function _artMapDecodeSmall(blob) { +// Decode to a sensible avatar size for how big the node actually draws — crisp +// where it matters (focal/watchlist nodes), light for the swarm of small ones — +// so total image memory stays ~150-250 MB instead of multiple GB. +function _artMapImgPx(px) { + return Math.min(384, Math.max(112, Math.round(px || 144))); +} +function _artMapDecodeSmall(blob, px) { if (!blob) return Promise.resolve(null); + const d = _artMapImgPx(px); try { return createImageBitmap(blob, { - resizeWidth: _ARTMAP_IMG_PX, resizeHeight: _ARTMAP_IMG_PX, resizeQuality: 'medium', + resizeWidth: d, resizeHeight: d, resizeQuality: 'high', }).catch(() => createImageBitmap(blob).catch(() => null)); // older engines ignore opts } catch (e) { return createImageBitmap(blob).catch(() => null); } } -function _artMapLoadImage(url) { +function _artMapLoadImage(url, px) { // Try direct CORS fetch first (zero server load, works for Spotify/iTunes/Discogs) return fetch(url, { mode: 'cors' }) .then(r => r.ok ? r.blob() : Promise.reject('not ok')) - .then(b => _artMapDecodeSmall(b)) + .then(b => _artMapDecodeSmall(b, px)) .catch(() => { // Fallback: server proxy for CDNs without CORS headers return fetch('/api/image-proxy?url=' + encodeURIComponent(url)) .then(r => r.ok ? r.blob() : null) - .then(b => _artMapDecodeSmall(b)) + .then(b => _artMapDecodeSmall(b, px)) .catch(() => null); }); } +// Target avatar px for a node, based on its world radius (≈ its on-screen size +// at full zoom). Focal/watchlist nodes get a big crisp avatar; small ones stay +// light. Used by every map's image loader. +function _artMapNodeImgPx(n) { + const isFocal = n.type === 'watchlist' || n.type === 'center' || n.ring === 1; + return _artMapImgPx(isFocal ? Math.max(256, (n.radius || 0) * 1.4) : (n.radius || 0) * 1.6); +} + function _artMapHideContextMenu() { const m = document.getElementById('artist-map-context'); if (m) m.style.display = 'none'; @@ -6762,7 +6793,8 @@ function _artMapSetupInteraction(canvas) { _artMapAnimateConstellation(); // fade out } if (_artMap.hoveredNode) { - // Delay constellation effect by 800ms of sustained hover + // Snappy sustained-hover delay before the constellation lights up + // (was 800ms, which felt like nothing happened). _artMap._constellationTimer = setTimeout(() => { if (_artMap.hoveredNode) { _artMap._constellationActive = true; @@ -6770,7 +6802,7 @@ function _artMapSetupInteraction(canvas) { _artMap._constellationCache = null; _artMapAnimateConstellation(); } - }, 800); + }, 220); } _artMapRender(); } From ff2bddf1db6e2c26e6f0ec3c6d41c78abecd179c Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 08:46:11 -0700 Subject: [PATCH 046/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20Phase?= =?UTF-8?q?=204:=20progressive=20image=20streaming=20(paint-first,=20click?= =?UTF-8?q?-click-click)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three maps (watchlist/genre/explore) now paint instantly with placeholder circles and stay fully interactive (pan/zoom/hover/click) while images stream in throttled ~280ms waves and sharpen the map in place. Replaces the old blocking 'await all N images then paint' loaders — the headline 'feels slow' fix. Focal/large nodes fetch first; a per-open load token cancels stale streams when you jump to another artist, so rapid click-through never piles up fetches. --- webui/static/discover.js | 140 +++++++++++++++++---------------------- 1 file changed, 59 insertions(+), 81 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index c40f30b9..827b6902 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5214,48 +5214,18 @@ async function openArtistMap() { setTimeout(async () => { _artMap.placed.forEach(n => { n.opacity = 1; }); - // Load images in parallel using createImageBitmap (non-blocking) - const loadingText = container.querySelector('.artist-map-loading-text'); - const imgNodes = _artMap.placed.filter(n => n.image_url); - let loaded = 0; - - if (loadingText) loadingText.textContent = `Loading ${imgNodes.length} artist images...`; - - // Batch image loading — 20 concurrent fetches - const CONCURRENT = 20; - let idx = 0; - - async function loadNextBatch() { - const batch = []; - while (idx < imgNodes.length && batch.length < CONCURRENT) { - const n = imgNodes[idx++]; - if (_artMap.images[n.id]) { loaded++; continue; } - batch.push( - _artMapLoadImage(n.image_url, _artMapNodeImgPx(n)) - .then(bmp => { if (bmp) _artMap.images[n.id] = bmp; }) - .finally(() => { - loaded++; - if (loadingText && loaded % 50 === 0) { - loadingText.textContent = `Loading images... ${loaded}/${imgNodes.length}`; - } - }) - ); - } - if (batch.length) await Promise.all(batch); - if (idx < imgNodes.length) return loadNextBatch(); - } - - await loadNextBatch(); - - // Build buffer and render - if (loadingText) loadingText.textContent = 'Rendering map...'; - await new Promise(r => setTimeout(r, 20)); // let text update render - + // Paint NOW — the map is fully interactive (pan/zoom/hover/click) + // with placeholder circles before a single image is fetched. Images + // then stream in throttled waves and sharpen the map in place. This + // is the "click-click-click" win: no blocking on N image fetches + // before the first frame. _artMap.dirty = true; _artMapRender(); const le = document.getElementById('artist-map-loading'); if (le) le.remove(); + + _artMapStreamImages(_artMap.placed); }, 50); } catch (err) { @@ -6250,32 +6220,15 @@ async function _openGenreMapWithSelection(selectedGenre) { // Load images + render if (loadingText) loadingText.textContent = `Rendering ${placedCount} artists...`; - setTimeout(async () => { - const imgNodes = _artMap.placed.filter(n => n.image_url && !n._isLabel); - let loaded = 0; - const CONCURRENT = 20; - let idx = 0; - async function loadBatch() { - const batch = []; - while (idx < imgNodes.length && batch.length < CONCURRENT) { - const n = imgNodes[idx++]; - batch.push(_artMapLoadImage(n.image_url, _artMapNodeImgPx(n)) - .then(bmp => { if (bmp) _artMap.images[n.id] = bmp; }) - .finally(() => { loaded++; })); - } - if (batch.length) await Promise.all(batch); - if (idx < imgNodes.length) return loadBatch(); - } - await loadBatch(); - _artMap.dirty = true; - _artMapRender(); - const le = document.getElementById('artist-map-loading'); - if (le) le.remove(); - }, 50); + const le = document.getElementById('artist-map-loading'); + if (le) le.remove(); _artMap.dirty = true; _artMapRender(); + // Stream images in throttled waves — interactive immediately, sharpens in place. + _artMapStreamImages(_artMap.placed.filter(n => !n._isLabel)); + } catch (err) { console.error('Genre map error:', err); const lt = container.querySelector('.artist-map-loading-text'); @@ -6477,32 +6430,15 @@ async function _openArtistMapExplorerWithName(name) { const loadingText = container.querySelector('.artist-map-loading-text'); if (loadingText) loadingText.textContent = `Loading ${_artMap.placed.length} artists...`; - setTimeout(async () => { - const imgNodes = _artMap.placed.filter(n => n.image_url); - let loaded = 0; - const CONCURRENT = 20; - let idx = 0; - async function loadBatch() { - const batch = []; - while (idx < imgNodes.length && batch.length < CONCURRENT) { - const n = imgNodes[idx++]; - batch.push(_artMapLoadImage(n.image_url, _artMapNodeImgPx(n)) - .then(bmp => { if (bmp) _artMap.images[n.id] = bmp; }) - .finally(() => { loaded++; })); - } - if (batch.length) await Promise.all(batch); - if (idx < imgNodes.length) return loadBatch(); - } - await loadBatch(); - _artMap.dirty = true; - _artMapRender(); - const le = document.getElementById('artist-map-loading'); - if (le) le.remove(); - }, 50); + const le = document.getElementById('artist-map-loading'); + if (le) le.remove(); _artMap.dirty = true; _artMapRender(); + // Stream images in throttled waves — interactive immediately, sharpens in place. + _artMapStreamImages(_artMap.placed); + } catch (err) { console.error('Artist explorer error:', err); const lt = container.querySelector('.artist-map-loading-text'); @@ -6668,6 +6604,48 @@ function _artMapNodeImgPx(n) { return _artMapImgPx(isFocal ? Math.max(256, (n.radius || 0) * 1.4) : (n.radius || 0) * 1.6); } +// Stream node images in the background WITHOUT blocking the first paint. The map +// is drawn immediately with placeholder circles and stays fully interactive +// (click/hover/pan) while images fill in. Redraws are throttled into ~waves so +// 1000s of arrivals don't trigger 1000s of buffer rebuilds. A load token makes +// opening another map cancel this stream (stale bitmaps are dropped). Focal +// nodes are fetched first so what you're looking at sharpens soonest. +function _artMapStreamImages(imgNodes, concurrent = 24) { + const token = (_artMap._loadToken = (_artMap._loadToken || 0) + 1); + // Focal/large nodes first — the user's eye lands there. + const queue = imgNodes.filter(n => n.image_url).slice().sort((a, b) => (b.radius || 0) - (a.radius || 0)); + let idx = 0, inFlight = 0, redrawPending = false; + + const scheduleRedraw = () => { + if (redrawPending || token !== _artMap._loadToken) return; + redrawPending = true; + setTimeout(() => { + redrawPending = false; + if (token !== _artMap._loadToken) return; + _artMap.dirty = true; + _artMapRender(); + }, 280); + }; + + function pump() { + if (token !== _artMap._loadToken) return; // a newer map took over + while (inFlight < concurrent && idx < queue.length) { + const n = queue[idx++]; + if (_artMap.images[n.id]) continue; + inFlight++; + _artMapLoadImage(n.image_url, _artMapNodeImgPx(n)) + .then(bmp => { + if (bmp && token === _artMap._loadToken) { + _artMap.images[n.id] = bmp; + scheduleRedraw(); + } + }) + .finally(() => { inFlight--; pump(); }); + } + } + pump(); +} + function _artMapHideContextMenu() { const m = document.getElementById('artist-map-context'); if (m) m.style.display = 'none'; From 8d4de4dc49ef6433ec1fa6e98abe8927f34025c2 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 08:54:32 -0700 Subject: [PATCH 047/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20Phase?= =?UTF-8?q?=205:=20incremental=20buffer=20compositing=20(no=20lag=20while?= =?UTF-8?q?=20images=20stream)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The streaming fix in Phase 4 still rebuilt the ENTIRE offscreen buffer (~1500 nodes) on each image wave, and any hover/pan during streaming hit that same dirty flag — so interacting while images loaded redrew the whole world over and over (the 'laggy until all images load' jank). Now each arriving image composites ONLY its own node into the existing buffer (_artMapCompositeNode) and does a cheap rAF-coalesced blit — no full rebuild. The per-node draw is extracted into _artMapDrawNodeToBuffer so the full rebuild and the incremental compositor share identical drawing (can't drift). Falls back to a full rebuild only if the buffer isn't built yet. Pan/hover stay at blit-speed the entire time images stream in. --- webui/static/discover.js | 242 ++++++++++++++++++++++----------------- 1 file changed, 139 insertions(+), 103 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index 827b6902..ddd9b646 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5370,108 +5370,7 @@ function _artMapRebuildBuffer() { else if (pass === 2 && !n._isLabel && (n.type === 'watchlist' || n.type === 'center' || n.ring === 1)) { /* draw */ } else continue; if (hideSimilar && n.type !== 'watchlist' && n.type !== 'center' && !n._isLabel) continue; - const op = n.opacity || 0; - if (op < 0.01) continue; - const r = n.radius; - const isW = n.type === 'watchlist' || n.type === 'center'; - octx.globalAlpha = op; - - // Genre label node — transparent circle with large text - if (n._isLabel) { - octx.globalAlpha = 0.6; - octx.beginPath(); - octx.arc(n.x, n.y, n.radius, 0, Math.PI * 2); - octx.fillStyle = 'rgba(138,43,226,0.04)'; - octx.fill(); - octx.strokeStyle = 'rgba(138,43,226,0.08)'; - octx.lineWidth = 1; - octx.stroke(); - const labelSize = Math.max(12, n.radius * 0.25); - octx.font = `800 ${labelSize}px system-ui`; - octx.textAlign = 'center'; - octx.textBaseline = 'middle'; - octx.fillStyle = 'rgba(138,43,226,0.35)'; - octx.fillText(n.name, n.x, n.y - labelSize * 0.3); - octx.font = `500 ${labelSize * 0.5}px system-ui`; - octx.fillStyle = 'rgba(255,255,255,0.15)'; - octx.fillText(`${n._count || 0} artists`, n.x, n.y + labelSize * 0.5); - octx.globalAlpha = 1; - continue; - } - - // Render quality based on node size in buffer pixels - const rScaled = r * scale; - const isSmall = rScaled < 8; - const isTiny = rScaled < 3; - - // Tiny nodes: just a colored dot (no clip, no image, no text) - if (isTiny) { - octx.beginPath(); - octx.arc(n.x, n.y, r, 0, Math.PI * 2); - octx.fillStyle = isW ? '#6b21a8' : '#2a2a40'; - octx.fill(); - continue; - } - - // Small nodes: filled circle + border, no image clip - if (isSmall) { - octx.beginPath(); - octx.arc(n.x, n.y, r, 0, Math.PI * 2); - const img = _artMap.images[n.id]; - if (img) { - octx.save(); octx.clip(); - octx.drawImage(img, n.x - r, n.y - r, r * 2, r * 2); - octx.restore(); - } else { - octx.fillStyle = isW ? '#1a0a30' : '#141420'; - octx.fill(); - } - octx.strokeStyle = isW ? 'rgba(138,43,226,0.3)' : 'rgba(255,255,255,0.06)'; - octx.lineWidth = isW ? 1.5 : 0.5; - octx.stroke(); - continue; - } - - // Full quality: glow + clip + image + text - if (isW) { - octx.beginPath(); - octx.arc(n.x, n.y, r + 4, 0, Math.PI * 2); - octx.strokeStyle = 'rgba(138,43,226,0.25)'; - octx.lineWidth = 5; - octx.stroke(); - } - - octx.save(); - octx.beginPath(); - octx.arc(n.x, n.y, r, 0, Math.PI * 2); - octx.closePath(); - octx.clip(); - - const img = _artMap.images[n.id]; - if (img) { - octx.drawImage(img, n.x - r, n.y - r, r * 2, r * 2); - octx.fillStyle = 'rgba(0,0,0,0.45)'; - octx.fillRect(n.x - r, n.y - r, r * 2, r * 2); - } else { - octx.fillStyle = isW ? '#1a0a30' : '#141420'; - octx.fillRect(n.x - r, n.y - r, r * 2, r * 2); - } - octx.restore(); - - octx.beginPath(); - octx.arc(n.x, n.y, r, 0, Math.PI * 2); - octx.strokeStyle = isW ? 'rgba(138,43,226,0.4)' : 'rgba(255,255,255,0.08)'; - octx.lineWidth = isW ? 2 : 0.5; - octx.stroke(); - - const fontSize = isW ? Math.max(16, r * 0.14) : Math.max(8, r * 0.3); - octx.font = `${isW ? '700' : '600'} ${fontSize}px system-ui`; - octx.textAlign = 'center'; - octx.textBaseline = 'middle'; - octx.fillStyle = '#fff'; - const maxC = isW ? 20 : 12; - const label = n.name.length > maxC ? n.name.substring(0, maxC - 1) + '…' : n.name; - octx.fillText(label, n.x, n.y); + _artMapDrawNodeToBuffer(octx, n, scale); } } @@ -5480,6 +5379,136 @@ function _artMapRebuildBuffer() { _artMap.dirty = false; } +// Draw a SINGLE node into the offscreen buffer (in world coords; caller has +// already applied the buffer's scale+translate). Shared by the full rebuild and +// the incremental image compositor so the two can never drift visually. +function _artMapDrawNodeToBuffer(octx, n, scale) { + const op = n.opacity || 0; + if (op < 0.01) return; + const r = n.radius; + const isW = n.type === 'watchlist' || n.type === 'center'; + octx.globalAlpha = op; + + // Genre label node — transparent circle with large text + if (n._isLabel) { + octx.globalAlpha = 0.6; + octx.beginPath(); + octx.arc(n.x, n.y, n.radius, 0, Math.PI * 2); + octx.fillStyle = 'rgba(138,43,226,0.04)'; + octx.fill(); + octx.strokeStyle = 'rgba(138,43,226,0.08)'; + octx.lineWidth = 1; + octx.stroke(); + const labelSize = Math.max(12, n.radius * 0.25); + octx.font = `800 ${labelSize}px system-ui`; + octx.textAlign = 'center'; + octx.textBaseline = 'middle'; + octx.fillStyle = 'rgba(138,43,226,0.35)'; + octx.fillText(n.name, n.x, n.y - labelSize * 0.3); + octx.font = `500 ${labelSize * 0.5}px system-ui`; + octx.fillStyle = 'rgba(255,255,255,0.15)'; + octx.fillText(`${n._count || 0} artists`, n.x, n.y + labelSize * 0.5); + octx.globalAlpha = 1; + return; + } + + // Render quality based on node size in buffer pixels + const rScaled = r * scale; + const isSmall = rScaled < 8; + const isTiny = rScaled < 3; + + // Tiny nodes: just a colored dot (no clip, no image, no text) + if (isTiny) { + octx.beginPath(); + octx.arc(n.x, n.y, r, 0, Math.PI * 2); + octx.fillStyle = isW ? '#6b21a8' : '#2a2a40'; + octx.fill(); + return; + } + + // Small nodes: filled circle + border, no image clip + if (isSmall) { + octx.beginPath(); + octx.arc(n.x, n.y, r, 0, Math.PI * 2); + const img = _artMap.images[n.id]; + if (img) { + octx.save(); octx.clip(); + octx.drawImage(img, n.x - r, n.y - r, r * 2, r * 2); + octx.restore(); + } else { + octx.fillStyle = isW ? '#1a0a30' : '#141420'; + octx.fill(); + } + octx.strokeStyle = isW ? 'rgba(138,43,226,0.3)' : 'rgba(255,255,255,0.06)'; + octx.lineWidth = isW ? 1.5 : 0.5; + octx.stroke(); + return; + } + + // Full quality: glow + clip + image + text + if (isW) { + octx.beginPath(); + octx.arc(n.x, n.y, r + 4, 0, Math.PI * 2); + octx.strokeStyle = 'rgba(138,43,226,0.25)'; + octx.lineWidth = 5; + octx.stroke(); + } + + octx.save(); + octx.beginPath(); + octx.arc(n.x, n.y, r, 0, Math.PI * 2); + octx.closePath(); + octx.clip(); + + const img = _artMap.images[n.id]; + if (img) { + octx.drawImage(img, n.x - r, n.y - r, r * 2, r * 2); + octx.fillStyle = 'rgba(0,0,0,0.45)'; + octx.fillRect(n.x - r, n.y - r, r * 2, r * 2); + } else { + octx.fillStyle = isW ? '#1a0a30' : '#141420'; + octx.fillRect(n.x - r, n.y - r, r * 2, r * 2); + } + octx.restore(); + + octx.beginPath(); + octx.arc(n.x, n.y, r, 0, Math.PI * 2); + octx.strokeStyle = isW ? 'rgba(138,43,226,0.4)' : 'rgba(255,255,255,0.08)'; + octx.lineWidth = isW ? 2 : 0.5; + octx.stroke(); + + const fontSize = isW ? Math.max(16, r * 0.14) : Math.max(8, r * 0.3); + octx.font = `${isW ? '700' : '600'} ${fontSize}px system-ui`; + octx.textAlign = 'center'; + octx.textBaseline = 'middle'; + octx.fillStyle = '#fff'; + const maxC = isW ? 20 : 12; + const label = n.name.length > maxC ? n.name.substring(0, maxC - 1) + '…' : n.name; + octx.fillText(label, n.x, n.y); +} + +// Composite ONE node into the EXISTING buffer without a full rebuild. This is +// what makes image streaming cheap: when a bitmap arrives we redraw only that +// node (in place, over its placeholder) instead of redrawing all ~1500 nodes. +// Returns false if there's no buffer yet (or the node is hidden) so the caller +// can fall back to a full rebuild. Zoom changes rebuild the buffer at a new +// scale; this always reads the CURRENT buffer scale/origin, so it stays correct. +function _artMapCompositeNode(n) { + const oc = _artMap.offscreen; + const scale = _artMap._bufferScale; + if (!oc || scale == null) return false; + if (_artMap._hideSimilar && n.type !== 'watchlist' && n.type !== 'center' && !n._isLabel) return false; + if ((n.opacity || 0) < 0.01) return false; + const octx = oc.getContext('2d'); + octx.save(); + octx.scale(scale, scale); + octx.translate(-_artMap._bufferMinX, -_artMap._bufferMinY); + _artMapDrawNodeToBuffer(octx, n, scale); + octx.restore(); + octx.globalAlpha = 1; + return true; +} + function _artMapRender() { // v2 perf: coalesce every render request into a single rAF, so a burst of // mousemove/pan/animation calls never draws more than once per frame. @@ -6616,6 +6645,8 @@ function _artMapStreamImages(imgNodes, concurrent = 24) { const queue = imgNodes.filter(n => n.image_url).slice().sort((a, b) => (b.radius || 0) - (a.radius || 0)); let idx = 0, inFlight = 0, redrawPending = false; + // Fallback full-rebuild path, used only if the buffer isn't ready yet + // (e.g. images returned before the first paint built the offscreen buffer). const scheduleRedraw = () => { if (redrawPending || token !== _artMap._loadToken) return; redrawPending = true; @@ -6637,7 +6668,12 @@ function _artMapStreamImages(imgNodes, concurrent = 24) { .then(bmp => { if (bmp && token === _artMap._loadToken) { _artMap.images[n.id] = bmp; - scheduleRedraw(); + // Composite just this node into the existing buffer (cheap) + // and blit. Only fall back to a full rebuild if the buffer + // isn't built yet. This keeps pan/hover at blit-speed the + // whole time images are streaming in — the lag fix. + if (_artMapCompositeNode(n)) _artMapRender(); + else scheduleRedraw(); } }) .finally(() => { inFlight--; pump(); }); From 4b04f765a0e9fe61b7e683f0848c6b3cf9f2167f Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 09:22:40 -0700 Subject: [PATCH 048/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20Phase?= =?UTF-8?q?=20A:=20two-layer=20live=20animation=20engine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation for the water/ripple redesign. Splits rendering into: - Static far-field buffer: small/distant bubbles, baked once (cheap blit). - Live overlay layer: every bubble big enough to read (radius*zoom >= LIVE_PX) redrawn each frame in world space, so it can scale/bob/ripple. Viewport- culled + capped at 600 draws. The partition is frozen at buffer-build zoom (_liveBuildZoom) so the two sets stay exact complements even mid-zoom — no flicker, no double-draw. Adds an idle-capable rAF loop (_artMapStartLoop/_artMapStepAnimations) that runs only while something animates and stops when still. First payload: a reveal — the far field fades in globally while live bubbles pop outward from the camera centre (ease-out-back, staggered by distance). Wired into all three loaders. Bonus: live bubbles now draw full-res at the current zoom instead of through the 4096px-capped buffer, so zoomed-in artwork is crisp (addresses the earlier low-res complaint structurally). Engine only — the island layout, ripple choreography and click physics build on this in B–E. 64 JS integrity tests pass. --- webui/static/discover.js | 187 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 180 insertions(+), 7 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index ddd9b646..41ce56d7 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5011,6 +5011,20 @@ const _artMap = { // under the LOD dot threshold). Only binds on large worlds — small maps // stay crisp via the z*2 / 1.0 caps. Tunable. MAX_BUFFER_PX: 4096, + + // ── v2 live-animation engine ────────────────────────────────────────── + // Two-layer render: the offscreen buffer holds the STATIC far field (small + // bubbles, drawn once); a LIVE overlay redraws only the bubbles big enough + // to read on screen, every frame, so they can scale/bob/ripple. A node is + // "live" when its on-screen radius (radius*zoom) clears LIVE_PX; everything + // smaller stays baked in the buffer. This bounds per-frame work to what you + // can actually see, not the full 2000-node world. + LIVE_PX: 12, + // The rAF loop runs ONLY while something is animating (reveal/ripple) and + // idles otherwise, falling back to on-demand blits. _anim tracks it. + _anim: { running: false, raf: null, last: 0 }, + _fieldAlpha: 1, // global fade for the static far-field buffer (reveal) + _revealT0: 0, // performance.now() when the current reveal began }; async function openArtistMap() { @@ -5215,16 +5229,15 @@ async function openArtistMap() { _artMap.placed.forEach(n => { n.opacity = 1; }); // Paint NOW — the map is fully interactive (pan/zoom/hover/click) - // with placeholder circles before a single image is fetched. Images - // then stream in throttled waves and sharpen the map in place. This - // is the "click-click-click" win: no blocking on N image fetches - // before the first frame. + // before a single image is fetched. The reveal animation blooms the + // bubbles in (far field fades, near bubbles pop outward); images then + // stream in behind it and sharpen in place. No blocking on N fetches. _artMap.dirty = true; - _artMapRender(); const le = document.getElementById('artist-map-loading'); if (le) le.remove(); + _artMapBeginReveal(); _artMapStreamImages(_artMap.placed); }, 50); @@ -5335,6 +5348,8 @@ function _artMapRebuildBuffer() { _artMap._bufferScale = scale; _artMap._bufferMinX = minX; _artMap._bufferMinY = minY; + // Freeze the live/buffer partition to this build's zoom (see _artMapIsLiveSize). + _artMap._liveBuildZoom = _artMap.zoom; octx.scale(scale, scale); octx.translate(-minX, -minY); @@ -5370,6 +5385,7 @@ function _artMapRebuildBuffer() { else if (pass === 2 && !n._isLabel && (n.type === 'watchlist' || n.type === 'center' || n.ring === 1)) { /* draw */ } else continue; if (hideSimilar && n.type !== 'watchlist' && n.type !== 'center' && !n._isLabel) continue; + if (_artMapIsLiveSize(n)) continue; // big enough to read → drawn live on the overlay _artMapDrawNodeToBuffer(octx, n, scale); } } @@ -5499,6 +5515,10 @@ function _artMapCompositeNode(n) { if (!oc || scale == null) return false; if (_artMap._hideSimilar && n.type !== 'watchlist' && n.type !== 'center' && !n._isLabel) return false; if ((n.opacity || 0) < 0.01) return false; + // Live-layer bubbles aren't in the buffer — they read their image fresh each + // frame, so just signal "blit" and let the overlay pick it up. Compositing + // them here would double-draw (buffer copy + live copy). + if (_artMapIsLiveSize(n)) return true; const octx = oc.getContext('2d'); octx.save(); octx.scale(scale, scale); @@ -5509,6 +5529,151 @@ function _artMapCompositeNode(n) { return true; } +// A node renders on the live overlay when it's big enough on screen to read. +// Labels stay baked in the static buffer (no per-frame motion needed in v2). +// IMPORTANT: this uses the zoom the BUFFER WAS BUILT AT (_liveBuildZoom), not +// the live zoom. The buffer only rebuilds ~300ms after zooming stops, so the +// live/buffer split must stay frozen to whatever the (possibly stale) buffer +// excluded — otherwise a bubble could fall out of both during an active zoom +// and flicker. Both the buffer-exclude and the live-draw read this same value, +// so the two sets are always exact complements. +function _artMapIsLiveSize(n) { + if (n._isLabel) return false; + const z = _artMap._liveBuildZoom || _artMap.zoom; + return (n.radius || 0) * z >= _artMap.LIVE_PX; +} + +// Draw the live overlay: every big/near bubble, in world space, honouring its +// per-node animation transform (aScale for reveal/pop, opacity for fade). Kept +// cheap by viewport culling + a hard cap; the static far field is already on +// screen via the buffer blit. +function _artMapDrawLiveLayer(ctx) { + const placed = _artMap.placed; + if (!placed || !placed.length) return; + const z = _artMap.zoom; + const ox = _artMap.offsetX, oy = _artMap.offsetY; + const w = _artMap.width, h = _artMap.height; + const margin = 80; + + ctx.save(); + ctx.translate(ox, oy); + ctx.scale(z, z); + + let drawn = 0; + const CAP = 600; + for (const n of placed) { + if (!_artMapIsLiveSize(n)) continue; + if (_artMap._hideSimilar && n.type !== 'watchlist' && n.type !== 'center' && !n._isLabel) continue; + // Viewport cull (screen space) + const sx = ox + n.x * z, sy = oy + n.y * z; + const rPx = (n.radius || 0) * z; + if (sx + rPx < -margin || sx - rPx > w + margin || sy + rPx < -margin || sy - rPx > h + margin) continue; + _artMapDrawLiveNode(ctx, n); + if (++drawn >= CAP) break; + } + ctx.restore(); + ctx.globalAlpha = 1; +} + +// Draw one live bubble with its animation transform. aScale scales about the +// node centre; opacity fades it. Reuses the shared node painter so the live +// bubble is pixel-identical to its baked form once settled. +function _artMapDrawLiveNode(ctx, n) { + const sc = n.aScale == null ? 1 : n.aScale; + if (sc <= 0.001) return; + if (sc !== 1) { + ctx.save(); + ctx.translate(n.x, n.y); + ctx.scale(sc, sc); + ctx.translate(-n.x, -n.y); + _artMapDrawNodeToBuffer(ctx, n, _artMap.zoom); + ctx.restore(); + } else { + _artMapDrawNodeToBuffer(ctx, n, _artMap.zoom); + } + ctx.globalAlpha = 1; +} + +// ── Animation loop ──────────────────────────────────────────────────────── +// Runs only while something is animating; idles otherwise. Each tick advances +// the active animations (reveal field-fade + per-node pop), draws one frame, +// and re-arms itself only if work remains — so a still map costs nothing. +function _artMapStartLoop() { + const a = _artMap._anim; + if (a.running) return; + a.running = true; + a.last = performance.now(); + const tick = (t) => { + if (!a.running) return; + const more = _artMapStepAnimations(t); + _artMapDraw(); + if (more) { + a.raf = requestAnimationFrame(tick); + } else { + a.running = false; + a.raf = null; + } + }; + a.raf = requestAnimationFrame(tick); +} + +// Advance every active animation by absolute time t (ms). Returns true while +// anything is still moving. Phase A: global field fade-in + staggered per-node +// reveal pop (ease-out-back) for the live bubbles. +function _artMapStepAnimations(t) { + let active = false; + + // Global far-field fade-in over the reveal window. + if (_artMap._fieldAlpha < 1) { + const FIELD_MS = 700; + const p = Math.min(1, (t - _artMap._revealT0) / FIELD_MS); + _artMap._fieldAlpha = p; + if (p < 1) active = true; + } + + // Per-node reveal pop: each node eases its aScale 0→1 once past its + // staggered start time. Ease-out-back gives a subtle overshoot ("bloom"). + const placed = _artMap.placed; + if (placed) { + for (const n of placed) { + if (n.aScale == null || n.aScale >= 1) continue; + const start = n._revealAt || _artMap._revealT0; + if (t < start) { active = true; continue; } + const DUR = 520; + const p = Math.min(1, (t - start) / DUR); + // ease-out-back + const c1 = 1.70158, c3 = c1 + 1; + const e = 1 + c3 * Math.pow(p - 1, 3) + c1 * Math.pow(p - 1, 2); + n.aScale = p >= 1 ? 1 : e; + if (p < 1) active = true; + } + } + + return active; +} + +// Kick off the reveal: fade the far field in, and stagger each live bubble's +// pop by its distance from the camera centre so islands bloom outward. +function _artMapBeginReveal() { + const t0 = performance.now(); + _artMap._revealT0 = t0; + _artMap._fieldAlpha = 0; + const cx = (_artMap.width / 2 - _artMap.offsetX) / _artMap.zoom; + const cy = (_artMap.height / 2 - _artMap.offsetY) / _artMap.zoom; + let maxD = 1; + for (const n of _artMap.placed) { + n._d2 = (n.x - cx) * (n.x - cx) + (n.y - cy) * (n.y - cy); + if (n._d2 > maxD) maxD = n._d2; + } + const maxD2 = Math.sqrt(maxD) || 1; + for (const n of _artMap.placed) { + n.aScale = 0; + const d = Math.sqrt(n._d2); + n._revealAt = t0 + (d / maxD2) * 520; // farther = later → outward bloom + } + _artMapStartLoop(); +} + function _artMapRender() { // v2 perf: coalesce every render request into a single rAF, so a burst of // mousemove/pan/animation calls never draws more than once per frame. @@ -5560,12 +5725,20 @@ function _artMapDraw() { // Screen position of world (wx,wy) = offsetX + wx*zoom, offsetY + wy*zoom // Therefore buffer origin on screen = offsetX + minX*zoom, offsetY + minY*zoom // And buffer is drawn at size (bufferWidth * zoom/s, bufferHeight * zoom/s) + const fieldAlpha = _artMap._fieldAlpha == null ? 1 : _artMap._fieldAlpha; + if (fieldAlpha < 0.999) ctx.globalAlpha = fieldAlpha; ctx.drawImage(oc, _artMap.offsetX + mx * z, _artMap.offsetY + my * z, oc.width * z / s, oc.height * z / s ); + ctx.globalAlpha = 1; + + // ── Live overlay layer: redraw the big/near bubbles every frame so they can + // scale (reveal/pop), bob and ripple. These are excluded from the static + // buffer above, so there's no double-draw. Viewport-culled + capped. ── + _artMapDrawLiveLayer(ctx); // ── Interactive overlay (drawn on main canvas, not buffer) ── const cFade = _artMap._constellationFade || 0; @@ -6253,7 +6426,7 @@ async function _openGenreMapWithSelection(selectedGenre) { if (le) le.remove(); _artMap.dirty = true; - _artMapRender(); + _artMapBeginReveal(); // Stream images in throttled waves — interactive immediately, sharpens in place. _artMapStreamImages(_artMap.placed.filter(n => !n._isLabel)); @@ -6463,7 +6636,7 @@ async function _openArtistMapExplorerWithName(name) { if (le) le.remove(); _artMap.dirty = true; - _artMapRender(); + _artMapBeginReveal(); // Stream images in throttled waves — interactive immediately, sharpens in place. _artMapStreamImages(_artMap.placed); From 0f63024d4b5242f115a341c05db07515ae4cb11f Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 09:37:17 -0700 Subject: [PATCH 049/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20Phase?= =?UTF-8?q?=20A.1:=20pre-masked=20circular=20images,=20consistent=20art,?= =?UTF-8?q?=20clean=20reveal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the screenshot feedback (mix of detailed covers + blank dots, lag, 'weird' load): - Pre-mask each album image into a circle ONCE at load (a canvas), so every draw is a plain drawImage instead of a per-frame ctx.clip(). Clipping was the live-layer stutter — hundreds of clips per frame. Now free. - Draw album art at nearly every on-screen size (only sub-2.2px fall back to a dot), instead of detailed-vs-blank-dot tiers. Consistent 'sea of covers'. - Reveal is now a clean ease-out-cubic fade of the whole map (buffer blit + live layer ramp together via _drawAlphaMul) — dropped the bouncy per-node pop that read as 'weird'. The real island ripple bloom comes in Phase C. 64 JS integrity tests pass. --- webui/static/discover.js | 170 ++++++++++++++++++--------------------- 1 file changed, 79 insertions(+), 91 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index 41ce56d7..7cfe0091 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5350,6 +5350,7 @@ function _artMapRebuildBuffer() { _artMap._bufferMinY = minY; // Freeze the live/buffer partition to this build's zoom (see _artMapIsLiveSize). _artMap._liveBuildZoom = _artMap.zoom; + _artMap._drawAlphaMul = 1; // buffer bakes at full alpha; the blit applies the reveal fade octx.scale(scale, scale); octx.translate(-minX, -minY); @@ -5403,11 +5404,14 @@ function _artMapDrawNodeToBuffer(octx, n, scale) { if (op < 0.01) return; const r = n.radius; const isW = n.type === 'watchlist' || n.type === 'center'; - octx.globalAlpha = op; + // Global fade multiplier (reveal). Lets the whole map fade in cleanly while + // each painter keeps its own per-element alpha. + const mul = _artMap._drawAlphaMul == null ? 1 : _artMap._drawAlphaMul; + octx.globalAlpha = op * mul; // Genre label node — transparent circle with large text if (n._isLabel) { - octx.globalAlpha = 0.6; + octx.globalAlpha = 0.6 * mul; octx.beginPath(); octx.arc(n.x, n.y, n.radius, 0, Math.PI * 2); octx.fillStyle = 'rgba(138,43,226,0.04)'; @@ -5428,13 +5432,14 @@ function _artMapDrawNodeToBuffer(octx, n, scale) { return; } - // Render quality based on node size in buffer pixels + // On-screen size drives detail. Album art shows at nearly every size (the + // images are pre-masked to circles, so this is just a cheap drawImage — no + // per-frame clip) for a consistent "sea of covers" look. Only the very + // smallest fall back to a coloured dot. const rScaled = r * scale; - const isSmall = rScaled < 8; - const isTiny = rScaled < 3; + const img = _artMap.images[n.id]; - // Tiny nodes: just a colored dot (no clip, no image, no text) - if (isTiny) { + if (rScaled < 2.2) { octx.beginPath(); octx.arc(n.x, n.y, r, 0, Math.PI * 2); octx.fillStyle = isW ? '#6b21a8' : '#2a2a40'; @@ -5442,27 +5447,8 @@ function _artMapDrawNodeToBuffer(octx, n, scale) { return; } - // Small nodes: filled circle + border, no image clip - if (isSmall) { - octx.beginPath(); - octx.arc(n.x, n.y, r, 0, Math.PI * 2); - const img = _artMap.images[n.id]; - if (img) { - octx.save(); octx.clip(); - octx.drawImage(img, n.x - r, n.y - r, r * 2, r * 2); - octx.restore(); - } else { - octx.fillStyle = isW ? '#1a0a30' : '#141420'; - octx.fill(); - } - octx.strokeStyle = isW ? 'rgba(138,43,226,0.3)' : 'rgba(255,255,255,0.06)'; - octx.lineWidth = isW ? 1.5 : 0.5; - octx.stroke(); - return; - } - - // Full quality: glow + clip + image + text - if (isW) { + // Focal glow ring for watchlist/center bubbles + if (isW && rScaled >= 7) { octx.beginPath(); octx.arc(n.x, n.y, r + 4, 0, Math.PI * 2); octx.strokeStyle = 'rgba(138,43,226,0.25)'; @@ -5470,37 +5456,43 @@ function _artMapDrawNodeToBuffer(octx, n, scale) { octx.stroke(); } - octx.save(); - octx.beginPath(); - octx.arc(n.x, n.y, r, 0, Math.PI * 2); - octx.closePath(); - octx.clip(); - - const img = _artMap.images[n.id]; + // Body — pre-masked circular image (no clip) or a placeholder disc. if (img) { octx.drawImage(img, n.x - r, n.y - r, r * 2, r * 2); - octx.fillStyle = 'rgba(0,0,0,0.45)'; - octx.fillRect(n.x - r, n.y - r, r * 2, r * 2); } else { + octx.beginPath(); + octx.arc(n.x, n.y, r, 0, Math.PI * 2); octx.fillStyle = isW ? '#1a0a30' : '#141420'; - octx.fillRect(n.x - r, n.y - r, r * 2, r * 2); + octx.fill(); } - octx.restore(); + const showLabel = rScaled >= 13; + + // Darken art behind the label so the name stays legible. + if (showLabel && img) { + octx.beginPath(); + octx.arc(n.x, n.y, r, 0, Math.PI * 2); + octx.fillStyle = 'rgba(0,0,0,0.42)'; + octx.fill(); + } + + // Border octx.beginPath(); octx.arc(n.x, n.y, r, 0, Math.PI * 2); - octx.strokeStyle = isW ? 'rgba(138,43,226,0.4)' : 'rgba(255,255,255,0.08)'; - octx.lineWidth = isW ? 2 : 0.5; + octx.strokeStyle = isW ? 'rgba(138,43,226,0.45)' : 'rgba(255,255,255,0.10)'; + octx.lineWidth = isW ? 2 : (rScaled >= 7 ? 1 : 0.5); octx.stroke(); - const fontSize = isW ? Math.max(16, r * 0.14) : Math.max(8, r * 0.3); - octx.font = `${isW ? '700' : '600'} ${fontSize}px system-ui`; - octx.textAlign = 'center'; - octx.textBaseline = 'middle'; - octx.fillStyle = '#fff'; - const maxC = isW ? 20 : 12; - const label = n.name.length > maxC ? n.name.substring(0, maxC - 1) + '…' : n.name; - octx.fillText(label, n.x, n.y); + if (showLabel) { + const fontSize = isW ? Math.max(16, r * 0.14) : Math.max(8, r * 0.3); + octx.font = `${isW ? '700' : '600'} ${fontSize}px system-ui`; + octx.textAlign = 'center'; + octx.textBaseline = 'middle'; + octx.fillStyle = '#fff'; + const maxC = isW ? 20 : 12; + const label = n.name.length > maxC ? n.name.substring(0, maxC - 1) + '…' : n.name; + octx.fillText(label, n.x, n.y); + } } // Composite ONE node into the EXISTING buffer without a full rebuild. This is @@ -5558,6 +5550,8 @@ function _artMapDrawLiveLayer(ctx) { ctx.save(); ctx.translate(ox, oy); ctx.scale(z, z); + // The live layer fades in together with the far field during the reveal. + _artMap._drawAlphaMul = _artMap._fieldAlpha == null ? 1 : _artMap._fieldAlpha; let drawn = 0; const CAP = 600; @@ -5571,6 +5565,7 @@ function _artMapDrawLiveLayer(ctx) { _artMapDrawLiveNode(ctx, n); if (++drawn >= CAP) break; } + _artMap._drawAlphaMul = 1; ctx.restore(); ctx.globalAlpha = 1; } @@ -5618,59 +5613,26 @@ function _artMapStartLoop() { } // Advance every active animation by absolute time t (ms). Returns true while -// anything is still moving. Phase A: global field fade-in + staggered per-node -// reveal pop (ease-out-back) for the live bubbles. +// anything is still moving. Phase A: a clean global fade-in (ease-out-cubic). +// The bubbly per-island ripple bloom is layered on in Phase C. function _artMapStepAnimations(t) { let active = false; - // Global far-field fade-in over the reveal window. if (_artMap._fieldAlpha < 1) { - const FIELD_MS = 700; + const FIELD_MS = 600; const p = Math.min(1, (t - _artMap._revealT0) / FIELD_MS); - _artMap._fieldAlpha = p; + _artMap._fieldAlpha = 1 - Math.pow(1 - p, 3); // ease-out-cubic if (p < 1) active = true; } - // Per-node reveal pop: each node eases its aScale 0→1 once past its - // staggered start time. Ease-out-back gives a subtle overshoot ("bloom"). - const placed = _artMap.placed; - if (placed) { - for (const n of placed) { - if (n.aScale == null || n.aScale >= 1) continue; - const start = n._revealAt || _artMap._revealT0; - if (t < start) { active = true; continue; } - const DUR = 520; - const p = Math.min(1, (t - start) / DUR); - // ease-out-back - const c1 = 1.70158, c3 = c1 + 1; - const e = 1 + c3 * Math.pow(p - 1, 3) + c1 * Math.pow(p - 1, 2); - n.aScale = p >= 1 ? 1 : e; - if (p < 1) active = true; - } - } - return active; } -// Kick off the reveal: fade the far field in, and stagger each live bubble's -// pop by its distance from the camera centre so islands bloom outward. +// Kick off the reveal: a clean fade-in of the whole map (far field via the +// buffer-blit alpha, live bubbles via _drawAlphaMul — they ramp together). function _artMapBeginReveal() { - const t0 = performance.now(); - _artMap._revealT0 = t0; + _artMap._revealT0 = performance.now(); _artMap._fieldAlpha = 0; - const cx = (_artMap.width / 2 - _artMap.offsetX) / _artMap.zoom; - const cy = (_artMap.height / 2 - _artMap.offsetY) / _artMap.zoom; - let maxD = 1; - for (const n of _artMap.placed) { - n._d2 = (n.x - cx) * (n.x - cx) + (n.y - cy) * (n.y - cy); - if (n._d2 > maxD) maxD = n._d2; - } - const maxD2 = Math.sqrt(maxD) || 1; - for (const n of _artMap.placed) { - n.aScale = 0; - const d = Math.sqrt(n._d2); - n._revealAt = t0 + (d / maxD2) * 520; // farther = later → outward bloom - } _artMapStartLoop(); } @@ -6778,9 +6740,35 @@ function _artMapDecodeSmall(blob, px) { try { return createImageBitmap(blob, { resizeWidth: d, resizeHeight: d, resizeQuality: 'high', - }).catch(() => createImageBitmap(blob).catch(() => null)); // older engines ignore opts + }).then(_artMapCircleMask) + .catch(() => createImageBitmap(blob).then(_artMapCircleMask).catch(() => null)); } catch (e) { - return createImageBitmap(blob).catch(() => null); + return createImageBitmap(blob).then(_artMapCircleMask).catch(() => null); + } +} + +// Pre-mask a decoded bitmap into a CIRCLE once, at load time, returning a +// canvas. The whole map then draws bubbles with a plain drawImage (the canvas +// is already round) instead of a per-frame ctx.clip() per bubble — clipping is +// one of the most expensive canvas ops and, at hundreds of visible bubbles per +// frame, was the live-layer stutter. Done once here, it's free forever after. +function _artMapCircleMask(src) { + if (!src) return null; + const w = src.width || 0; + if (!w) return src; + try { + const c = document.createElement('canvas'); + c.width = w; c.height = w; + const cx = c.getContext('2d'); + cx.beginPath(); + cx.arc(w / 2, w / 2, w / 2, 0, Math.PI * 2); + cx.closePath(); + cx.clip(); + cx.drawImage(src, 0, 0, w, w); + if (src.close) src.close(); // free the ImageBitmap; we keep the canvas + return c; + } catch (e) { + return src; // fall back to the raw bitmap (draw path still clips defensively) } } From f311ff0440beb2427832e802bf769dbe2c823d13 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 09:48:18 -0700 Subject: [PATCH 050/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20Phase?= =?UTF-8?q?=20B:=20genre=20islands=20everywhere?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three maps (watchlist / genre / explore) now lay out as genre 'islands' on the water via one shared engine (_artMapLayoutIslands): - Group artists by primary genre (long tail folds into 'Other'; max 14 islands). - Each island is a FILLED disc of covers packed centre-out (no empty donut hole), most-popular nearest the middle, focal artists sized up + centre-most. - Islands spread by golden spiral + push-apart with generous water between. - Clean floating genre TITLE above each island (hue-tinted, glow) instead of the old giant translucent label bubble. - Per-genre accent hue tints member-bubble borders so clusters read as a family. - Discovery edges (watchlist→similar, center→ring1→ring2) remapped to the new node ids so the hover constellation still works across islands. Replaces the per-artist donut clusters from the screenshots. Shared helpers: _artMapGroupByGenre, _artMapPackDisc, _artMapRemapEdges, _artMapFitToContent. 64 JS integrity tests pass. --- webui/static/discover.js | 617 +++++++++++++-------------------------- 1 file changed, 207 insertions(+), 410 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index 7cfe0091..e097d773 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5027,6 +5027,164 @@ const _artMap = { _revealT0: 0, // performance.now() when the current reveal began }; +// ── Genre-island layout (shared by watchlist / genre / explore) ─────────── +// Every map groups its artists into genre "islands" floating on the water: +// each island is a FILLED disc of album covers (packed centre-out, no empty +// donut hole), with a floating genre title above it and a per-genre accent hue. +// Islands are spread by a golden spiral + push-apart with generous spacing. + +// Deterministic hue (0–360) from a genre name, so each island has a stable tint. +function _artMapGenreHue(name) { + let h = 0; + const s = (name || '').toLowerCase(); + for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) % 360; + return h; +} + +// Pack members into a filled disc, centre outward. Returns relative offsets and +// the disc radius. Most-popular members land nearest the centre. +function _artMapPackDisc(members, nodeR, gap) { + const placements = []; + if (!members.length) return { placements, islandR: nodeR }; + placements.push({ node: members[0], dx: 0, dy: 0 }); + let idx = 1, ring = 1, ringDist = nodeR * 2 + gap; + const step = nodeR * 2 + gap; + while (idx < members.length) { + const circ = 2 * Math.PI * ringDist; + const cap = Math.max(1, Math.floor(circ / step)); + const cnt = Math.min(cap, members.length - idx); + const aStep = (2 * Math.PI) / cnt; + const off = ring * 2.399963; // golden offset per ring → no spokes + for (let i = 0; i < cnt; i++) { + const a = off + i * aStep; + placements.push({ node: members[idx + i], dx: Math.cos(a) * ringDist, dy: Math.sin(a) * ringDist }); + } + idx += cnt; + ringDist += step; + ring++; + } + return { placements, islandR: ringDist }; +} + +// Group flat nodes by primary genre into {name, count, nodes[]}, largest first. +// Nodes with no genre fall into "Other". Focal nodes (watchlist/center) keep +// their flag so the layout can size them up. +function _artMapGroupByGenre(nodes, maxIslands = 14) { + const byGenre = {}; + for (const n of nodes) { + const g = (n.genres && n.genres.length) ? String(n.genres[0]) : 'Other'; + const key = g.replace(/\b\w/g, c => c.toUpperCase()); + (byGenre[key] = byGenre[key] || []).push(n); + } + let groups = Object.keys(byGenre).map(name => ({ name, nodes: byGenre[name], count: byGenre[name].length })); + groups.sort((a, b) => b.count - a.count); + if (groups.length > maxIslands) { + // Fold the long tail of tiny genres into one "Other" island. + const head = groups.slice(0, maxIslands - 1); + const tail = groups.slice(maxIslands - 1); + const tailNodes = tail.flatMap(g => g.nodes); + head.push({ name: 'Other', nodes: tailNodes, count: tailNodes.length }); + groups = head; + } + return groups; +} + +// Lay out islands → fills _artMap.placed and _artMap._islands. groups is an +// array of {name, count, nodes[]} where each node has name/image_url/genres/ids. +function _artMapLayoutIslands(groups, opts = {}) { + _artMap.placed = []; + _artMap._islands = []; + const nodeR = opts.nodeR || (_artMap.WATCHLIST_R * 0.22); + const gap = opts.gap || (_artMap.BUFFER * 2.2); + const cap = opts.maxPerIsland || 500; + let pid = 0; + + const islands = groups.map(g => { + const members = g.nodes.slice() + .sort((a, b) => (b._focal ? 1 : 0) - (a._focal ? 1 : 0) || (b.popularity || 0) - (a.popularity || 0)) + .slice(0, cap); + const { placements, islandR } = _artMapPackDisc(members, nodeR, gap); + return { name: g.name, count: g.count != null ? g.count : members.length, placements, islandR, hue: _artMapGenreHue(g.name) }; + }); + + // Golden-spiral seed placement + islands.forEach((isl, i) => { + if (i === 0) { isl.cx = 0; isl.cy = 0; } + else { const a = i * 2.399963; const r = isl.islandR * Math.sqrt(i) * 1.05; isl.cx = Math.cos(a) * r; isl.cy = Math.sin(a) * r; } + }); + // Push apart — generous water between islands + for (let pass = 0; pass < 160; pass++) { + let moved = false; + for (let i = 0; i < islands.length; i++) { + for (let j = i + 1; j < islands.length; j++) { + const dx = islands[j].cx - islands[i].cx, dy = islands[j].cy - islands[i].cy; + const dist = Math.hypot(dx, dy) || 1; + const minD = islands[i].islandR + islands[j].islandR + nodeR * 7; + if (dist < minD) { + const push = (minD - dist) / 2 + 1; + islands[i].cx -= dx / dist * push; islands[i].cy -= dy / dist * push; + islands[j].cx += dx / dist * push; islands[j].cy += dy / dist * push; + moved = true; + } + } + } + if (!moved) break; + } + + for (const isl of islands) { + // Floating genre title above the island + _artMap.placed.push({ + id: `label_${isl.name}`, name: isl.name, x: isl.cx, y: isl.cy - isl.islandR - nodeR * 1.4, + radius: Math.max(nodeR * 1.3, isl.islandR * 0.16), opacity: 1, + type: 'genre_label', _isLabel: true, _count: isl.count, _hue: isl.hue, image_url: '', + }); + for (const p of isl.placements) { + const n = p.node; + _artMap.placed.push({ + id: pid++, _origId: n.id, name: n.name, + x: isl.cx + p.dx, y: isl.cy + p.dy, + radius: nodeR * (n._focal ? 1.45 : 1), opacity: 1, + type: n.type || 'similar', + image_url: n.image_url || '', genres: n.genres || [], + spotify_id: n.spotify_id || '', itunes_id: n.itunes_id || '', + deezer_id: n.deezer_id || '', discogs_id: n.discogs_id || '', + musicbrainz_id: n.musicbrainz_id || '', + popularity: n.popularity || 0, _hue: isl.hue, _island: isl.name, + }); + } + _artMap._islands.push({ name: isl.name, cx: isl.cx, cy: isl.cy, r: isl.islandR, hue: isl.hue, count: isl.count }); + } + + _artMap._nodeById = {}; + _artMap.placed.forEach(n => { _artMap._nodeById[n.id] = n; }); +} + +// Remap edges that used original node ids to the new placed-node ids. +function _artMapRemapEdges(edges) { + const map = {}; + for (const n of _artMap.placed) if (n._origId != null && map[n._origId] == null) map[n._origId] = n.id; + const out = []; + for (const e of (edges || [])) { + const s = map[e.source], t = map[e.target]; + if (s != null && t != null && s !== t) out.push({ source: s, target: t, weight: e.weight || 1 }); + } + return out; +} + +// Auto-zoom/pan so all placed nodes fit the viewport with a margin. +function _artMapFitToContent(marginPx = 120) { + let minX = Infinity, maxX = -Infinity, minY = Infinity, maxY = -Infinity; + for (const n of _artMap.placed) { + minX = Math.min(minX, n.x - n.radius); maxX = Math.max(maxX, n.x + n.radius); + minY = Math.min(minY, n.y - n.radius); maxY = Math.max(maxY, n.y + n.radius); + } + if (!isFinite(minX)) return; + const mapW = maxX - minX + marginPx * 2, mapH = maxY - minY + marginPx * 2; + _artMap.zoom = Math.min(_artMap.width / mapW, _artMap.height / mapH, 1); + _artMap.offsetX = _artMap.width / 2 - ((minX + maxX) / 2) * _artMap.zoom; + _artMap.offsetY = _artMap.height / 2 - ((minY + maxY) / 2) * _artMap.zoom; +} + async function openArtistMap() { const container = document.getElementById('artist-map-container'); if (!container) return; @@ -5073,141 +5231,15 @@ async function openArtistMap() { document.getElementById('artist-map-stats').textContent = `${data.watchlist_count} watchlist · ${data.similar_count} similar`; - _artMap.edges = data.edges; - const WR = _artMap.WATCHLIST_R; - const BUF = _artMap.BUFFER; - - // ── PHASE 1: Place watchlist artists with guaranteed no-overlap ── - const wNodes = data.nodes.filter(n => n.type === 'watchlist'); - // Minimum center-to-center distance between watchlist nodes - const minCenterDist = WR * 3.5; // WR*2 for radii + WR*1.5 gap — similar artists fill the gaps via spiral packing - - // Place watchlist nodes in a spiral — deterministic, guaranteed spacing - wNodes.forEach((n, i) => { - n.radius = WR; - n.opacity = 0; - if (i === 0) { - n.x = 0; n.y = 0; - } else { - // Golden angle spiral for even distribution - const angle = i * 2.399963; // golden angle in radians - const r = minCenterDist * Math.sqrt(i) * 0.7; - n.x = Math.cos(angle) * r; - n.y = Math.sin(angle) * r; - } - }); - - // Post-process: push apart any watchlist nodes that ended up too close - for (let pass = 0; pass < 50; pass++) { - let moved = false; - for (let i = 0; i < wNodes.length; i++) { - for (let j = i + 1; j < wNodes.length; j++) { - const dx = wNodes[j].x - wNodes[i].x; - const dy = wNodes[j].y - wNodes[i].y; - const dist = Math.sqrt(dx * dx + dy * dy) || 1; - if (dist < minCenterDist) { - const push = (minCenterDist - dist) / 2 + 1; - const nx = (dx / dist) * push; - const ny = (dy / dist) * push; - wNodes[i].x -= nx; wNodes[i].y -= ny; - wNodes[j].x += nx; wNodes[j].y += ny; - moved = true; - } - } - } - if (!moved) break; - } - - wNodes.forEach(n => { _artMap.placed.push(n); }); - - // ── PHASE 2: Place similar artists around their source watchlist nodes ── - const sNodes = data.nodes.filter(n => n.type === 'similar'); - sNodes.forEach(n => { - const occ = n.occurrence || 1; - const rank = n.rank || 5; - // Bigger overall: min 25% of WR, max 55%, scaled by relevance - n.radius = Math.min(WR * 0.55, Math.max(WR * 0.25, WR * 0.2 + occ * WR * 0.06 + (10 - rank) * WR * 0.025)); - }); - sNodes.sort((a, b) => b.radius - a.radius); - - // Build edge lookup: target_id → source node (O(1) instead of .find()) - const edgeMap = {}; - _artMap.edges.forEach(e => { edgeMap[e.target] = e.source; }); - const nodeById = {}; - _artMap.placed.forEach(n => { nodeById[n.id] = n; }); - - // Spatial grid for fast collision detection - // Cell size must cover the largest possible bubble diameter + buffer - const CELL = WR * 2 + BUF * 2; - const grid = {}; - function _gridKey(x, y) { return `${Math.floor(x / CELL)},${Math.floor(y / CELL)}`; } - function _gridAdd(n) { - const k = _gridKey(n.x, n.y); - if (!grid[k]) grid[k] = []; - grid[k].push(n); - } - function _gridCheck(x, y, r) { - const cx = Math.floor(x / CELL); - const cy = Math.floor(y / CELL); - // Search wider radius to catch large watchlist bubbles - for (let dx = -3; dx <= 3; dx++) { - for (let dy = -3; dy <= 3; dy++) { - const cell = grid[`${cx + dx},${cy + dy}`]; - if (!cell) continue; - for (const p of cell) { - const ddx = x - p.x, ddy = y - p.y; - const minD = r + p.radius + BUF; - if (ddx * ddx + ddy * ddy < minD * minD) return true; - } - } - } - return false; - } - // Add watchlist nodes to grid - _artMap.placed.forEach(n => _gridAdd(n)); - - // Place similar nodes with spatial grid collision - for (const sn of sNodes) { - const srcId = edgeMap[sn.id]; - const src = srcId != null ? nodeById[srcId] : null; - const cx = src ? src.x : 0; - const cy = src ? src.y : 0; - const startDist = (src ? src.radius : WR) + sn.radius + BUF; - - let placed = false; - for (let dist = startDist; dist < startDist + WR * 3; dist += sn.radius * 0.5) { - const steps = Math.max(8, Math.floor(dist * 0.1)); - const off = Math.random() * Math.PI * 2; - for (let a = 0; a < steps; a++) { - const angle = off + (a / steps) * Math.PI * 2; - const tx = cx + Math.cos(angle) * dist; - const ty = cy + Math.sin(angle) * dist; - if (!_gridCheck(tx, ty, sn.radius)) { - sn.x = tx; sn.y = ty; sn.opacity = 0; - _artMap.placed.push(sn); - nodeById[sn.id] = sn; - _gridAdd(sn); - placed = true; - break; - } - } - if (placed) break; - } - } - - // Auto-zoom to fit all nodes - let minX = Infinity, maxX = -Infinity, minY = Infinity, maxY = -Infinity; - _artMap.placed.forEach(n => { - minX = Math.min(minX, n.x - n.radius); - maxX = Math.max(maxX, n.x + n.radius); - minY = Math.min(minY, n.y - n.radius); - maxY = Math.max(maxY, n.y + n.radius); - }); - const mapW = maxX - minX + 200; - const mapH = maxY - minY + 200; - _artMap.zoom = Math.min(_artMap.width / mapW, _artMap.height / mapH, 1); - _artMap.offsetX = _artMap.width / 2 - ((minX + maxX) / 2) * _artMap.zoom; - _artMap.offsetY = _artMap.height / 2 - ((minY + maxY) / 2) * _artMap.zoom; + // Group every watchlist + similar artist into genre islands. Watchlist + // artists are focal (sit centre-most + sized up). The discovery edges + // (watchlist → similar) are remapped to the new node ids so the hover + // constellation still shows who's related across islands. + const rawNodes = data.nodes.map(n => ({ ...n, _focal: n.type === 'watchlist' })); + const groups = _artMapGroupByGenre(rawNodes); + _artMapLayoutIslands(groups); + _artMap.edges = _artMapRemapEdges(data.edges); + _artMapFitToContent(); // Setup interaction _artMapSetupInteraction(canvas); @@ -5409,25 +5441,26 @@ function _artMapDrawNodeToBuffer(octx, n, scale) { const mul = _artMap._drawAlphaMul == null ? 1 : _artMap._drawAlphaMul; octx.globalAlpha = op * mul; - // Genre label node — transparent circle with large text + // Genre title — a clean floating label above its island (no big bubble). if (n._isLabel) { - octx.globalAlpha = 0.6 * mul; - octx.beginPath(); - octx.arc(n.x, n.y, n.radius, 0, Math.PI * 2); - octx.fillStyle = 'rgba(138,43,226,0.04)'; - octx.fill(); - octx.strokeStyle = 'rgba(138,43,226,0.08)'; - octx.lineWidth = 1; - octx.stroke(); - const labelSize = Math.max(12, n.radius * 0.25); - octx.font = `800 ${labelSize}px system-ui`; + const hue = n._hue == null ? 270 : n._hue; + const titleSize = Math.max(13, n.radius * 0.42); + const name = (n.name || '').toUpperCase(); octx.textAlign = 'center'; octx.textBaseline = 'middle'; - octx.fillStyle = 'rgba(138,43,226,0.35)'; - octx.fillText(n.name, n.x, n.y - labelSize * 0.3); - octx.font = `500 ${labelSize * 0.5}px system-ui`; - octx.fillStyle = 'rgba(255,255,255,0.15)'; - octx.fillText(`${n._count || 0} artists`, n.x, n.y + labelSize * 0.5); + // Soft glow behind the title for legibility over the water. + octx.globalAlpha = mul; + octx.font = `800 ${titleSize}px system-ui, sans-serif`; + octx.shadowColor = `hsla(${hue},70%,12%,0.9)`; + octx.shadowBlur = titleSize * 0.6; + octx.fillStyle = `hsla(${hue},85%,82%,0.96)`; + octx.fillText(name, n.x, n.y); + octx.shadowBlur = 0; + // Count subtitle + octx.globalAlpha = 0.55 * mul; + octx.font = `600 ${titleSize * 0.42}px system-ui, sans-serif`; + octx.fillStyle = 'rgba(255,255,255,0.7)'; + octx.fillText(`${n._count || 0} artists`, n.x, n.y + titleSize * 0.85); octx.globalAlpha = 1; return; } @@ -5476,10 +5509,12 @@ function _artMapDrawNodeToBuffer(octx, n, scale) { octx.fill(); } - // Border + // Border — tinted with the island's genre hue so clusters read as a family. octx.beginPath(); octx.arc(n.x, n.y, r, 0, Math.PI * 2); - octx.strokeStyle = isW ? 'rgba(138,43,226,0.45)' : 'rgba(255,255,255,0.10)'; + if (isW) octx.strokeStyle = 'rgba(138,43,226,0.5)'; + else if (n._hue != null) octx.strokeStyle = `hsla(${n._hue},70%,70%,0.22)`; + else octx.strokeStyle = 'rgba(255,255,255,0.10)'; octx.lineWidth = isW ? 2 : (rScaled >= 7 ? 1 : 0.5); octx.stroke(); @@ -6230,154 +6265,17 @@ async function _openGenreMapWithSelection(selectedGenre) { document.getElementById('artist-map-stats').innerHTML = `${escapeHtml(selectedGenre)} ▾ · ${genres.length} genre${genres.length > 1 ? 's' : ''} · ${totalArtists} artists`; - const WR = _artMap.WATCHLIST_R; - const BUF = _artMap.BUFFER; - - const maxPerGenre = 500; - const nodeR = WR * 0.2; - - // Calculate actual cluster radius for each genre based on ring count - function getClusterRadius(artistCount) { - const count = Math.min(artistCount, maxPerGenre); - let ringDist = WR + nodeR * 2 + BUF; - let placed = 0; - while (placed < count) { - const circ = 2 * Math.PI * ringDist; - const inRing = Math.max(1, Math.floor(circ / (nodeR * 2 + BUF))); - placed += Math.min(inRing, count - placed); - ringDist += nodeR * 2 + BUF; - } - return ringDist; - } - - // Pre-compute cluster radii - genres.forEach(g => { g._clusterR = getClusterRadius(g.artist_ids.length); }); - - // Golden spiral placement - genres.forEach((g, i) => { - if (i === 0) { g._cx = 0; g._cy = 0; } - else { - const angle = i * 2.399963; - const r = g._clusterR * Math.sqrt(i) * 0.9; - g._cx = Math.cos(angle) * r; - g._cy = Math.sin(angle) * r; - } - }); - - // Push apart using actual cluster radii — no overlap possible - for (let pass = 0; pass < 80; pass++) { - let moved = false; - for (let i = 0; i < genres.length; i++) { - for (let j = i + 1; j < genres.length; j++) { - const dx = genres[j]._cx - genres[i]._cx; - const dy = genres[j]._cy - genres[i]._cy; - const dist = Math.sqrt(dx * dx + dy * dy) || 1; - const minDist = genres[i]._clusterR + genres[j]._clusterR + BUF * 4; - if (dist < minDist) { - const push = (minDist - dist) / 2 + 1; - genres[i]._cx -= (dx / dist) * push; genres[i]._cy -= (dy / dist) * push; - genres[j]._cx += (dx / dist) * push; genres[j]._cy += (dy / dist) * push; - moved = true; - } - } - } - if (!moved) break; - } - - let placedCount = 0; - - // Place genre labels as big watchlist-style bubbles - for (const g of genres) { - _artMap.placed.push({ - id: `genre_${g.name}`, name: g.name.toUpperCase(), - x: g._cx, y: g._cy, radius: WR, opacity: 1, - type: 'genre_label', image_url: '', genres: [g.name], - _isLabel: true, _count: g.count - }); - } - - // Place artists in concentric rings — deterministic O(1) per node, handles 10K+ instantly - let genreIdx = 0; - - async function placeGenreArtists() { - for (; genreIdx < genres.length; genreIdx++) { - const genre = genres[genreIdx]; - const artists = genre.artist_ids.slice(0, maxPerGenre); - const sorted = artists.map(nid => data.nodes[nid]).filter(Boolean).sort((a, b) => (b.popularity || 0) - (a.popularity || 0)); - - let ringDist = WR + nodeR * 2 + BUF; - let ringNum = 0; - let placed = 0; - - while (placed < sorted.length) { - const circumference = 2 * Math.PI * ringDist; - const nodesInRing = Math.max(1, Math.floor(circumference / (nodeR * 2 + BUF))); - const count = Math.min(nodesInRing, sorted.length - placed); - const angleStep = (2 * Math.PI) / nodesInRing; - const angleOffset = ringNum * 0.618; - - for (let i = 0; i < count; i++) { - const n = sorted[placed + i]; - if (!n) continue; - const isW = n.type === 'watchlist' || n.type === 'center'; - const r = isW ? nodeR * 1.5 : nodeR; - const angle = angleOffset + i * angleStep; - - _artMap.placed.push({ - id: placedCount + 1000, _origId: n.id, name: n.name, - x: genre._cx + Math.cos(angle) * ringDist, - y: genre._cy + Math.sin(angle) * ringDist, - radius: r, opacity: 1, - type: isW ? 'watchlist' : 'similar', - image_url: n.image_url || '', genres: n.genres || [], - spotify_id: n.spotify_id || '', itunes_id: n.itunes_id || '', - deezer_id: n.deezer_id || '', discogs_id: n.discogs_id || '', - }); - placedCount++; - } - placed += count; - ringDist += nodeR * 2 + BUF; - ringNum++; - } - - if (loadingText) loadingText.textContent = `Placing artists... ${genreIdx + 1}/${genres.length} genres (${placedCount} placed)`; - if (genreIdx % 5 === 0) await new Promise(r => setTimeout(r, 0)); - } - } - await placeGenreArtists(); - - // Build edges: connect artists that appear in multiple genre clusters + // Build genre-island groups from the selected + related genres and lay + // them out as filled-disc islands on the water (shared engine). + const groups = genres.map(g => ({ + name: g.name, + count: g.count, + nodes: (g.artist_ids || []).map(nid => data.nodes[nid]).filter(Boolean), + })); + _artMapLayoutIslands(groups); _artMap.edges = []; - const artistNodes = {}; - _artMap.placed.forEach(n => { - if (n._origId != null) { - if (!artistNodes[n._origId]) artistNodes[n._origId] = []; - artistNodes[n._origId].push(n.id); - } - }); - Object.values(artistNodes).forEach(ids => { - if (ids.length > 1) { - for (let i = 0; i < ids.length - 1; i++) { - _artMap.edges.push({ source: ids[i], target: ids[i + 1], weight: 5 }); - } - } - }); - - _artMap._nodeById = {}; - _artMap.placed.forEach(n => { _artMap._nodeById[n.id] = n; }); - - // Auto-zoom - let minX = Infinity, maxX = -Infinity, minY = Infinity, maxY = -Infinity; - _artMap.placed.forEach(n => { - minX = Math.min(minX, n.x - n.radius); - maxX = Math.max(maxX, n.x + n.radius); - minY = Math.min(minY, n.y - n.radius); - maxY = Math.max(maxY, n.y + n.radius); - }); - const mapW = maxX - minX + 200, mapH = maxY - minY + 200; - _artMap.zoom = Math.min(_artMap.width / mapW, _artMap.height / mapH, 1); - _artMap.offsetX = _artMap.width / 2 - ((minX + maxX) / 2) * _artMap.zoom; - _artMap.offsetY = _artMap.height / 2 - ((minY + maxY) / 2) * _artMap.zoom; + _artMapFitToContent(); + const placedCount = _artMap.placed.filter(n => !n._isLabel).length; _artMapSetupInteraction(canvas); @@ -6477,116 +6375,15 @@ async function _openArtistMapExplorerWithName(name) { document.getElementById('artist-map-stats').textContent = `${data.center} · ${ring1Count} similar · ${ring2Count} extended`; - _artMap.edges = data.edges; - const WR = _artMap.WATCHLIST_R; - const BUF = _artMap.BUFFER; - - // Layout: center node at origin, ring 1 in circle around it, ring 2 around ring 1 - const centerNode = data.nodes[0]; - centerNode.x = 0; centerNode.y = 0; - centerNode.radius = WR * 1.2; // Extra large center - centerNode.opacity = 1; - centerNode.type = 'center'; - _artMap.placed.push(centerNode); - - const CELL = WR * 2 + BUF * 2; - const grid = {}; - function _gk(x, y) { return `${Math.floor(x / CELL)},${Math.floor(y / CELL)}`; } - function _ga(n) { const k = _gk(n.x, n.y); if (!grid[k]) grid[k] = []; grid[k].push(n); } - function _gc(x, y, r) { - const cx = Math.floor(x / CELL), cy = Math.floor(y / CELL); - for (let dx = -3; dx <= 3; dx++) for (let dy = -3; dy <= 3; dy++) { - const cell = grid[`${cx + dx},${cy + dy}`]; - if (!cell) continue; - for (const p of cell) { - const ddx = x - p.x, ddy = y - p.y; - if (ddx * ddx + ddy * ddy < (r + p.radius + BUF) * (r + p.radius + BUF)) return true; - } - } - return false; - } - _ga(centerNode); - - // Place ring 1 in a circle - const ring1 = data.nodes.filter(n => n.ring === 1); - const ring1Dist = WR * 2.5; - ring1.forEach((n, i) => { - const angle = (i / ring1.length) * Math.PI * 2; - const rank = n.rank || 5; - n.radius = WR * 0.4 + (10 - rank) * WR * 0.03; - n.opacity = 1; - - // Find non-colliding position near ideal - let placed = false; - for (let dist = ring1Dist; dist < ring1Dist + WR * 3; dist += n.radius * 0.5) { - for (let ao = 0; ao < 6; ao++) { - const a = angle + (ao * 0.1 * (ao % 2 ? 1 : -1)); - const tx = Math.cos(a) * dist; - const ty = Math.sin(a) * dist; - if (!_gc(tx, ty, n.radius)) { - n.x = tx; n.y = ty; - _artMap.placed.push(n); - _ga(n); - placed = true; - break; - } - } - if (placed) break; - } - }); - - // Place ring 2 around their ring 1 sources - const ring2 = data.nodes.filter(n => n.ring === 2); - const nodeById = {}; - _artMap.placed.forEach(n => { nodeById[n.id] = n; }); - - ring2.forEach(n => { - // Find the ring 1 node that connects to this - const edge = data.edges.find(e => e.target === n.id); - const src = edge ? nodeById[edge.source] : null; - const cx = src ? src.x : 0; - const cy = src ? src.y : 0; - - n.radius = WR * 0.2 + (n.popularity || 0) / 100 * WR * 0.1; - n.opacity = 1; - - const startDist = (src ? src.radius : WR) + n.radius + BUF; - let placed = false; - for (let dist = startDist; dist < startDist + WR * 2; dist += n.radius * 0.5) { - const steps = Math.max(8, Math.floor(dist * 0.08)); - const off = Math.random() * Math.PI * 2; - for (let a = 0; a < steps; a++) { - const angle = off + (a / steps) * Math.PI * 2; - const tx = cx + Math.cos(angle) * dist; - const ty = cy + Math.sin(angle) * dist; - if (!_gc(tx, ty, n.radius)) { - n.x = tx; n.y = ty; - _artMap.placed.push(n); - _ga(n); - placed = true; - break; - } - } - if (placed) break; - } - }); - - // Build node lookup for edges - _artMap._nodeById = {}; - _artMap.placed.forEach(n => { _artMap._nodeById[n.id] = n; }); - - // Auto-zoom - let minX = Infinity, maxX = -Infinity, minY = Infinity, maxY = -Infinity; - _artMap.placed.forEach(n => { - minX = Math.min(minX, n.x - n.radius); - maxX = Math.max(maxX, n.x + n.radius); - minY = Math.min(minY, n.y - n.radius); - maxY = Math.max(maxY, n.y + n.radius); - }); - const mapW = maxX - minX + 200, mapH = maxY - minY + 200; - _artMap.zoom = Math.min(_artMap.width / mapW, _artMap.height / mapH, 1); - _artMap.offsetX = _artMap.width / 2 - ((minX + maxX) / 2) * _artMap.zoom; - _artMap.offsetY = _artMap.height / 2 - ((minY + maxY) / 2) * _artMap.zoom; + // Group the center + all discovered artists into genre islands. The + // center artist is focal. Discovery edges (center → similar → extended) + // are remapped so the hover constellation still traces how you got from + // one artist to another across the islands. + const rawNodes = data.nodes.map(n => ({ ...n, _focal: n.ring === 0 || n.type === 'center' })); + const groups = _artMapGroupByGenre(rawNodes); + _artMapLayoutIslands(groups); + _artMap.edges = _artMapRemapEdges(data.edges); + _artMapFitToContent(); _artMapSetupInteraction(canvas); From 9cce7810ea23fc4b6fb385a7791ffa29927e1f53 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 09:52:38 -0700 Subject: [PATCH 051/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20Phase?= =?UTF-8?q?=20C:=20ripple-bloom=20reveal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Islands now bloom in like drops on water instead of a flat fade: - Each island reveals in turn (staggered by island order); within an island, bubbles fade + scale (0.55→1, ease-out) outward from the centre by radial distance — a drop-in-water bloom. Genre titles fade in just after. - A hue-tinted water ripple ring expands from each island centre as it blooms (_artMapDrawRipples — reused by click ripples in Phase E). - During the reveal the static buffer is bypassed so EVERY bubble can animate (live layer, cap 2200); when the bloom ends it bakes into the buffer once and steady-state returns to the cheap two-layer path. - aAlpha folds into the global draw-alpha multiplier so fades compose cleanly. 64 JS integrity tests pass. --- webui/static/discover.js | 168 ++++++++++++++++++++++++++++----------- 1 file changed, 123 insertions(+), 45 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index e097d773..57407b6f 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5585,13 +5585,16 @@ function _artMapDrawLiveLayer(ctx) { ctx.save(); ctx.translate(ox, oy); ctx.scale(z, z); - // The live layer fades in together with the far field during the reveal. - _artMap._drawAlphaMul = _artMap._fieldAlpha == null ? 1 : _artMap._fieldAlpha; + _artMap._drawAlphaMul = 1; + // During the reveal the buffer is bypassed, so the live layer draws EVERY + // bubble (and the genre titles) so they can all bloom. Otherwise it draws + // only the big/near ones; the rest live in the static buffer. + const revealing = _artMap._revealing; let drawn = 0; - const CAP = 600; + const CAP = revealing ? 2200 : 600; for (const n of placed) { - if (!_artMapIsLiveSize(n)) continue; + if (!revealing && !_artMapIsLiveSize(n)) continue; if (_artMap._hideSimilar && n.type !== 'watchlist' && n.type !== 'center' && !n._isLabel) continue; // Viewport cull (screen space) const sx = ox + n.x * z, sy = oy + n.y * z; @@ -5606,11 +5609,14 @@ function _artMapDrawLiveLayer(ctx) { } // Draw one live bubble with its animation transform. aScale scales about the -// node centre; opacity fades it. Reuses the shared node painter so the live -// bubble is pixel-identical to its baked form once settled. +// node centre; aAlpha fades it (folded into the global draw-alpha multiplier). +// Reuses the shared node painter so the bubble is identical to its baked form +// once settled. function _artMapDrawLiveNode(ctx, n) { const sc = n.aScale == null ? 1 : n.aScale; if (sc <= 0.001) return; + const baseMul = _artMap._drawAlphaMul == null ? 1 : _artMap._drawAlphaMul; + _artMap._drawAlphaMul = baseMul * (n.aAlpha == null ? 1 : n.aAlpha); if (sc !== 1) { ctx.save(); ctx.translate(n.x, n.y); @@ -5621,6 +5627,7 @@ function _artMapDrawLiveNode(ctx, n) { } else { _artMapDrawNodeToBuffer(ctx, n, _artMap.zoom); } + _artMap._drawAlphaMul = baseMul; ctx.globalAlpha = 1; } @@ -5648,29 +5655,99 @@ function _artMapStartLoop() { } // Advance every active animation by absolute time t (ms). Returns true while -// anything is still moving. Phase A: a clean global fade-in (ease-out-cubic). -// The bubbly per-island ripple bloom is layered on in Phase C. +// anything is still moving. Phase C: each bubble scales+fades in (ease-out) +// once past its staggered start, and a water ripple expands from each island. function _artMapStepAnimations(t) { let active = false; - if (_artMap._fieldAlpha < 1) { - const FIELD_MS = 600; - const p = Math.min(1, (t - _artMap._revealT0) / FIELD_MS); - _artMap._fieldAlpha = 1 - Math.pow(1 - p, 3); // ease-out-cubic - if (p < 1) active = true; + const placed = _artMap.placed; + if (placed) { + for (const n of placed) { + if (n.aScale == null || n.aScale >= 1) continue; + if (t < n._revealAt) { active = true; continue; } + const p = Math.min(1, (t - n._revealAt) / (n._revealDur || 480)); + const e = 1 - Math.pow(1 - p, 3); // ease-out-cubic + if (p >= 1) { n.aScale = 1; n.aAlpha = 1; } + else { n.aScale = 0.55 + 0.45 * e; n.aAlpha = e; active = true; } + } } + const rip = _artMap._ripples; + if (rip && rip.length) { + let anyAlive = false; + for (const r of rip) if (t < r.t0 + r.dur) anyAlive = true; + if (anyAlive) active = true; else _artMap._ripples = []; + } + + // When the bloom finishes, leave reveal mode and bake everything into the + // static buffer (one rebuild) so steady-state goes back to the cheap path. + if (!active && _artMap._revealing) { + _artMap._revealing = false; + _artMap.dirty = true; + return true; // one more frame to do the rebuild + final blit + } return active; } -// Kick off the reveal: a clean fade-in of the whole map (far field via the -// buffer-blit alpha, live bubbles via _drawAlphaMul — they ramp together). +// Kick off the ripple-bloom reveal. Each island blooms in turn (staggered by +// island order); within an island, bubbles fade+scale outward from the centre +// like a drop hitting water, and a ripple ring expands from each island centre. +// During the reveal the whole map renders on the live layer (the static buffer +// is bypassed) so every bubble can animate; it bakes into the buffer at the end. function _artMapBeginReveal() { - _artMap._revealT0 = performance.now(); - _artMap._fieldAlpha = 0; + const t0 = performance.now(); + _artMap._revealT0 = t0; + _artMap._revealing = true; + _artMap._fieldAlpha = 1; // buffer is bypassed while revealing; live layer draws all + + const islands = _artMap._islands || []; + const islByName = {}; + islands.forEach((isl, i) => { isl._order = i; islByName[isl.name] = isl; }); + + const ISL_STAGGER = 145, RADIAL_MS = 430, NODE_DUR = 470; + for (const n of _artMap.placed) { + n.aScale = 0; n.aAlpha = 0; + const isl = islByName[n._island] || (n._isLabel ? islByName[n.name] : null); + const order = isl ? isl._order : 0; + let radial = 0; + if (isl && isl.r > 0) radial = Math.min(1, Math.hypot(n.x - isl.cx, n.y - isl.cy) / isl.r); + n._revealAt = t0 + order * ISL_STAGGER + radial * RADIAL_MS + (n._isLabel ? 90 : 0); + n._revealDur = NODE_DUR; + } + + _artMap._ripples = islands.map(isl => ({ + cx: isl.cx, cy: isl.cy, hue: isl.hue, + maxR: isl.r * 1.45, t0: t0 + isl._order * ISL_STAGGER, dur: 1150, + })); + _artMapStartLoop(); } +// Draw the expanding water-ripple rings (reveal + click). Cheap stroked arcs, +// hue-tinted, fading as they grow. Drawn in world space with a screen-constant +// line width. +function _artMapDrawRipples(ctx) { + const rip = _artMap._ripples; + if (!rip || !rip.length) return; + const t = performance.now(); + const z = _artMap.zoom; + ctx.save(); + ctx.translate(_artMap.offsetX, _artMap.offsetY); + ctx.scale(z, z); + for (const r of rip) { + const p = (t - r.t0) / r.dur; + if (p < 0 || p > 1) continue; + const radius = r.maxR * (0.08 + 0.92 * (1 - Math.pow(1 - p, 2))); + const alpha = (1 - p) * 0.55; + ctx.beginPath(); + ctx.arc(r.cx, r.cy, radius, 0, Math.PI * 2); + ctx.strokeStyle = `hsla(${r.hue},85%,72%,${alpha})`; + ctx.lineWidth = (1.5 + 6 * (1 - p)) / z; // ~constant on screen + ctx.stroke(); + } + ctx.restore(); +} + function _artMapRender() { // v2 perf: coalesce every render request into a single rAF, so a burst of // mousemove/pan/animation calls never draws more than once per frame. @@ -5704,38 +5781,39 @@ function _artMapDraw() { ctx.fillStyle = _artMap._bgGrad; ctx.fillRect(0, 0, w, h); - if (_artMap.dirty || !_artMap.offscreen) { - const _rt = _artMap._perf ? performance.now() : 0; - _artMapRebuildBuffer(); - if (_artMap._perf) _artMap._rebuildMs = performance.now() - _rt; - } - if (!_artMap.offscreen) { if (_artMap._perf) _artMapDrawPerf(ctx, _t0); return; } - - const oc = _artMap.offscreen; - const s = _artMap._bufferScale; - const mx = _artMap._bufferMinX; - const my = _artMap._bufferMinY; const z = _artMap.zoom; - // Blit offscreen buffer: the buffer was drawn with scale(s) + translate(-minX,-minY) - // So buffer pixel (bx,by) corresponds to world (bx/s + minX, by/s + minY) - // Screen position of world (wx,wy) = offsetX + wx*zoom, offsetY + wy*zoom - // Therefore buffer origin on screen = offsetX + minX*zoom, offsetY + minY*zoom - // And buffer is drawn at size (bufferWidth * zoom/s, bufferHeight * zoom/s) - const fieldAlpha = _artMap._fieldAlpha == null ? 1 : _artMap._fieldAlpha; - if (fieldAlpha < 0.999) ctx.globalAlpha = fieldAlpha; - ctx.drawImage(oc, - _artMap.offsetX + mx * z, - _artMap.offsetY + my * z, - oc.width * z / s, - oc.height * z / s - ); - ctx.globalAlpha = 1; + // While the ripple-bloom reveal is running, bypass the static buffer + // entirely and let the live layer draw every bubble (so each can animate). + // The buffer is (re)built once when the reveal ends. + if (!_artMap._revealing) { + if (_artMap.dirty || !_artMap.offscreen) { + const _rt = _artMap._perf ? performance.now() : 0; + _artMapRebuildBuffer(); + if (_artMap._perf) _artMap._rebuildMs = performance.now() - _rt; + } + if (_artMap.offscreen) { + const oc = _artMap.offscreen; + const s = _artMap._bufferScale; + const mx = _artMap._bufferMinX; + const my = _artMap._bufferMinY; + // Blit offscreen buffer (built with scale(s) + translate(-minX,-minY)). + const fieldAlpha = _artMap._fieldAlpha == null ? 1 : _artMap._fieldAlpha; + if (fieldAlpha < 0.999) ctx.globalAlpha = fieldAlpha; + ctx.drawImage(oc, + _artMap.offsetX + mx * z, + _artMap.offsetY + my * z, + oc.width * z / s, + oc.height * z / s + ); + ctx.globalAlpha = 1; + } + } - // ── Live overlay layer: redraw the big/near bubbles every frame so they can - // scale (reveal/pop), bob and ripple. These are excluded from the static - // buffer above, so there's no double-draw. Viewport-culled + capped. ── + // ── Live overlay layer: big/near bubbles every frame (during the reveal, + // ALL bubbles) so they can scale/bob/ripple. Viewport-culled + capped. ── _artMapDrawLiveLayer(ctx); + _artMapDrawRipples(ctx); // ── Interactive overlay (drawn on main canvas, not buffer) ── const cFade = _artMap._constellationFade || 0; From 33c4eea2010bc7a2eb5259feac0c5affee6e5477 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 10:13:05 -0700 Subject: [PATCH 052/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20Phase?= =?UTF-8?q?=20D:=20ambient=20buoyancy=20+=20glassy=20bubbles=20+=20tighter?= =?UTF-8?q?=20islands?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Ambient bob: bubbles gently float (sine offset, phase varies by position so they move in a wave, not in unison). Driven by a persistent rAF loop that runs only while bubbles are on screen + the tab is visible, and parks when zoomed out (_liveCount==0) or the map closes — so an idle overview costs nothing. - Glassy specular highlight (cached sprite, cheap drawImage per bubble) so bubbles read as glossy orbs at every size. - Tighter island spacing (water gap 7*nodeR → 3.5*nodeR) so the settled overview is more substantial, not thin-spread — addresses the 'mini version' feel after the reveal ripples fade. - Ambient resumes on zoom and on tab re-focus; stops cleanly on close. 64 JS integrity tests pass. --- webui/static/discover.js | 76 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index 57407b6f..c0842bfc 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5033,6 +5033,32 @@ const _artMap = { // donut hole), with a floating genre title above it and a per-genre accent hue. // Islands are spread by a golden spiral + push-apart with generous spacing. +// A cached circular "gloss" sprite — a soft top-left specular highlight that, +// drawn over each bubble, makes it read as a glassy orb. One radial gradient, +// rendered once; per-bubble it's just a cheap drawImage (no per-frame gradient). +function _artMapGlossSprite() { + if (_artMap._gloss) return _artMap._gloss; + const S = 128; + const c = document.createElement('canvas'); + c.width = S; c.height = S; + const cx = c.getContext('2d'); + cx.beginPath(); cx.arc(S / 2, S / 2, S / 2, 0, Math.PI * 2); cx.clip(); + const g = cx.createRadialGradient(S * 0.34, S * 0.28, S * 0.02, S * 0.5, S * 0.5, S * 0.62); + g.addColorStop(0, 'rgba(255,255,255,0.40)'); + g.addColorStop(0.32, 'rgba(255,255,255,0.10)'); + g.addColorStop(0.6, 'rgba(255,255,255,0.0)'); + cx.fillStyle = g; + cx.fillRect(0, 0, S, S); + // A faint inner-bottom shade for roundness + const g2 = cx.createRadialGradient(S * 0.5, S * 0.78, S * 0.05, S * 0.5, S * 0.5, S * 0.7); + g2.addColorStop(0, 'rgba(0,0,0,0.18)'); + g2.addColorStop(0.5, 'rgba(0,0,0,0.0)'); + cx.fillStyle = g2; + cx.fillRect(0, 0, S, S); + _artMap._gloss = c; + return c; +} + // Deterministic hue (0–360) from a genre name, so each island has a stable tint. function _artMapGenreHue(name) { let h = 0; @@ -5119,7 +5145,7 @@ function _artMapLayoutIslands(groups, opts = {}) { for (let j = i + 1; j < islands.length; j++) { const dx = islands[j].cx - islands[i].cx, dy = islands[j].cy - islands[i].cy; const dist = Math.hypot(dx, dy) || 1; - const minD = islands[i].islandR + islands[j].islandR + nodeR * 7; + const minD = islands[i].islandR + islands[j].islandR + nodeR * 3.5; if (dist < minD) { const push = (minD - dist) / 2 + 1; islands[i].cx -= dx / dist * push; islands[i].cy -= dy / dist * push; @@ -5150,6 +5176,10 @@ function _artMapLayoutIslands(groups, opts = {}) { deezer_id: n.deezer_id || '', discogs_id: n.discogs_id || '', musicbrainz_id: n.musicbrainz_id || '', popularity: n.popularity || 0, _hue: isl.hue, _island: isl.name, + // Ambient buoyancy — phase varies by position so bubbles bob in a + // gentle wave (not in unison); amplitude in world units. + _bobPhase: (isl.cx + p.dx + isl.cy + p.dy) * 0.0022, + _bobAmp: nodeR * 0.12, }); } _artMap._islands.push({ name: isl.name, cx: isl.cx, cy: isl.cy, r: isl.islandR, hue: isl.hue, count: isl.count }); @@ -5338,6 +5368,10 @@ function closeArtistMap() { const sidebar = document.getElementById('artmap-genre-sidebar'); if (sidebar) sidebar.style.display = 'none'; if (_artMap.animFrame) cancelAnimationFrame(_artMap.animFrame); + // Stop the ambient buoyancy loop so it doesn't run with the map hidden. + _artMap._ambient = false; + _artMap._anim.running = false; + if (_artMap._anim.raf) { cancelAnimationFrame(_artMap._anim.raf); _artMap._anim.raf = null; } if (_artMap._keyHandler) window.removeEventListener('keydown', _artMap._keyHandler); _artMapHideContextMenu(); @@ -5499,6 +5533,11 @@ function _artMapDrawNodeToBuffer(octx, n, scale) { octx.fill(); } + // Glassy specular highlight (orb look) — skip the tiniest where it'd be noise. + if (rScaled >= 5) { + octx.drawImage(_artMapGlossSprite(), n.x - r, n.y - r, r * 2, r * 2); + } + const showLabel = rScaled >= 13; // Darken art behind the label so the name stays legible. @@ -5606,6 +5645,9 @@ function _artMapDrawLiveLayer(ctx) { _artMap._drawAlphaMul = 1; ctx.restore(); ctx.globalAlpha = 1; + // Count of non-label bubbles drawn live — drives whether the ambient loop + // keeps running (zoomed out = 0 = loop parks). + _artMap._liveCount = revealing ? 0 : drawn; } // Draw one live bubble with its animation transform. aScale scales about the @@ -5617,9 +5659,14 @@ function _artMapDrawLiveNode(ctx, n) { if (sc <= 0.001) return; const baseMul = _artMap._drawAlphaMul == null ? 1 : _artMap._drawAlphaMul; _artMap._drawAlphaMul = baseMul * (n.aAlpha == null ? 1 : n.aAlpha); - if (sc !== 1) { + // Ambient buoyancy (steady state only — the reveal has its own motion). + let bobY = 0; + if (!_artMap._revealing && n._bobAmp) { + bobY = Math.sin((_artMap._now || 0) * 0.0016 + (n._bobPhase || 0)) * n._bobAmp; + } + if (sc !== 1 || bobY) { ctx.save(); - ctx.translate(n.x, n.y); + ctx.translate(n.x, n.y + bobY); ctx.scale(sc, sc); ctx.translate(-n.x, -n.y); _artMapDrawNodeToBuffer(ctx, n, _artMap.zoom); @@ -5642,9 +5689,14 @@ function _artMapStartLoop() { a.last = performance.now(); const tick = (t) => { if (!a.running) return; + _artMap._now = t; const more = _artMapStepAnimations(t); - _artMapDraw(); - if (more) { + _artMapDraw(); // sets _artMap._liveCount + // Keep looping while something is animating, OR while ambient buoyancy is + // on AND there are live bubbles on screen to bob (so a zoomed-out static + // overview costs nothing — the loop parks until the next interaction). + const keep = more || (_artMap._ambient && _artMap._liveCount > 0 && !document.hidden); + if (keep) { a.raf = requestAnimationFrame(tick); } else { a.running = false; @@ -5654,6 +5706,12 @@ function _artMapStartLoop() { a.raf = requestAnimationFrame(tick); } +// (Re)start the ambient loop if buoyancy is on and it isn't already running — +// called after the reveal and on zoom/pan so bob resumes when bubbles appear. +function _artMapEnsureAmbient() { + if (_artMap._ambient && !_artMap._anim.running && !document.hidden) _artMapStartLoop(); +} + // Advance every active animation by absolute time t (ms). Returns true while // anything is still moving. Phase C: each bubble scales+fades in (ease-out) // once past its staggered start, and a water ripple expands from each island. @@ -5698,6 +5756,7 @@ function _artMapBeginReveal() { const t0 = performance.now(); _artMap._revealT0 = t0; _artMap._revealing = true; + _artMap._ambient = true; // keep the loop alive afterwards for buoyancy _artMap._fieldAlpha = 1; // buffer is bypassed while revealing; live layer draws all const islands = _artMap._islands || []; @@ -5761,6 +5820,7 @@ function _artMapRender() { function _artMapDraw() { /**Blit offscreen buffer to screen canvas with pan/zoom. Near-zero cost.**/ const _t0 = _artMap._perf ? performance.now() : 0; + if (!_artMap._anim.running) _artMap._now = performance.now(); // keep bob current on on-demand draws const ctx = _artMap.ctx; const w = _artMap.width; const h = _artMap.height; @@ -6728,6 +6788,11 @@ function _artMapSetupInteraction(canvas) { if (canvas._artMapListenersAttached) return; canvas._artMapListenersAttached = true; + // Pause ambient buoyancy when the tab is hidden; resume on return. + document.addEventListener('visibilitychange', () => { + if (!document.hidden) _artMapEnsureAmbient(); + }); + let isPanning = false, panStartX = 0, panStartY = 0; canvas.addEventListener('wheel', (e) => { @@ -6742,6 +6807,7 @@ function _artMapSetupInteraction(canvas) { _artMap.offsetY = my - (my - _artMap.offsetY) * (newZoom / _artMap.zoom); _artMap.zoom = newZoom; _artMapRender(); // fast blit + _artMapEnsureAmbient(); // resume buoyancy if we zoomed bubbles into view // Debounce hi-res rebuild after zoom settles clearTimeout(_artMap._zoomRebuild); _artMap._zoomRebuild = setTimeout(() => { _artMap.dirty = true; _artMapRender(); }, 300); From cdc1f0bc107dc4a6ca59a1016166c1e631d87a02 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 10:16:27 -0700 Subject: [PATCH 053/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20Phase?= =?UTF-8?q?=20E:=20click=20ripple=20shoves=20nearby=20bubbles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clicking (or tapping) the map now drops a water ripple: a hue-tinted ring expands from the point AND nearby bubbles get shoved radially outward at the wavefront, then settle back as it passes and decays (_artMapNodeDisplacement — a gaussian bump at the expanding front, world-space radial push). Ripples emit from the clicked bubble's centre in its genre hue (or the bare click point), and still open the artist after a beat. Replaces the old single purple ring. Note: the physical shove acts on live-layer (zoomed-in) bubbles; at the far-out overview the ring shows but the tiny baked bubbles don't move. 64 tests pass. --- webui/static/discover.js | 87 ++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index c0842bfc..b4b1e4d8 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5659,14 +5659,17 @@ function _artMapDrawLiveNode(ctx, n) { if (sc <= 0.001) return; const baseMul = _artMap._drawAlphaMul == null ? 1 : _artMap._drawAlphaMul; _artMap._drawAlphaMul = baseMul * (n.aAlpha == null ? 1 : n.aAlpha); - // Ambient buoyancy (steady state only — the reveal has its own motion). - let bobY = 0; - if (!_artMap._revealing && n._bobAmp) { - bobY = Math.sin((_artMap._now || 0) * 0.0016 + (n._bobPhase || 0)) * n._bobAmp; + // Ambient buoyancy + ripple shove (steady state only — the reveal has its + // own motion). Both are world-space offsets applied about the node centre. + let ox = 0, oy = 0; + if (!_artMap._revealing) { + if (n._bobAmp) oy += Math.sin((_artMap._now || 0) * 0.0016 + (n._bobPhase || 0)) * n._bobAmp; + const disp = _artMapNodeDisplacement(n); + if (disp) { ox += disp.dx; oy += disp.dy; } } - if (sc !== 1 || bobY) { + if (sc !== 1 || ox || oy) { ctx.save(); - ctx.translate(n.x, n.y + bobY); + ctx.translate(n.x + ox, n.y + oy); ctx.scale(sc, sc); ctx.translate(-n.x, -n.y); _artMapDrawNodeToBuffer(ctx, n, _artMap.zoom); @@ -5807,6 +5810,43 @@ function _artMapDrawRipples(ctx) { ctx.restore(); } +// Total world-space displacement on a node from all active "push" ripples — the +// expanding wavefront shoves nearby bubbles radially outward, then they settle +// back as the wave passes and decays. Returns null when nothing is pushing. +function _artMapNodeDisplacement(n) { + const rip = _artMap._ripples; + if (!rip || !rip.length) return null; + const t = _artMap._now || performance.now(); + let dx = 0, dy = 0; + for (const r of rip) { + if (!r.push) continue; + const p = (t - r.t0) / r.dur; + if (p < 0 || p > 1) continue; + const front = r.maxR * (0.08 + 0.92 * (1 - Math.pow(1 - p, 2))); + const ddx = n.x - r.cx, ddy = n.y - r.cy; + const d = Math.hypot(ddx, ddy) || 1; + const delta = d - front; + const width = r.width || (r.maxR * 0.2); + const env = Math.exp(-(delta * delta) / (2 * width * width)); // bump at the wavefront + const push = r.push * env * (1 - p); // decays over the ripple's life + if (push > 0.05) { dx += (ddx / d) * push; dy += (ddy / d) * push; } + } + return (dx || dy) ? { dx, dy } : null; +} + +// Emit a water ripple at a world point — a fading ring plus a radial shove of +// nearby bubbles. Used for click/tap feedback. +function _artMapEmitRipple(wx, wy, hue) { + if (!_artMap._ripples) _artMap._ripples = []; + const WR = _artMap.WATCHLIST_R; + _artMap._ripples.push({ + cx: wx, cy: wy, hue: hue == null ? 270 : hue, + maxR: WR * 2.6, t0: performance.now(), dur: 900, + push: WR * 0.22, width: WR * 0.6, + }); + _artMapStartLoop(); // animate the ripple (guards against double-start) +} + function _artMapRender() { // v2 perf: coalesce every render request into a single rAF, so a burst of // mousemove/pan/animation calls never draws more than once per frame. @@ -6017,28 +6057,6 @@ function _artMapDraw() { ctx.restore(); } - // Click ripple animation - if (_artMap._ripple) { - const rip = _artMap._ripple; - const elapsed = performance.now() - rip.start; - const progress = elapsed / 400; // 400ms duration - if (progress < 1) { - ctx.save(); - ctx.translate(_artMap.offsetX, _artMap.offsetY); - ctx.scale(z, z); - const ripR = rip.radius + rip.radius * progress * 0.5; - ctx.beginPath(); - ctx.arc(rip.x, rip.y, ripR, 0, Math.PI * 2); - ctx.strokeStyle = `rgba(138,43,226,${0.5 * (1 - progress)})`; - ctx.lineWidth = 3 * (1 - progress); - ctx.stroke(); - ctx.restore(); - requestAnimationFrame(() => _artMapRender()); - } else { - _artMap._ripple = null; - } - } - if (_artMap._perf) _artMapDrawPerf(ctx, _t0); } @@ -6931,15 +6949,13 @@ function _artMapSetupInteraction(canvas) { isPanning = false; if (!wasDrag && clickStart) { - // It was a click — find the node under cursor + // It was a click — emit a water ripple (shoves nearby bubbles) and, + // if it landed on an artist, open it after the ripple kicks off. const { nx, ny } = _artMapScreenToWorld(e, canvas); const node = _artMapHitTest(nx, ny); - if (node) { - _artMap._ripple = { x: node.x, y: node.y, radius: node.radius, start: performance.now() }; - _artMapRender(); - if (node.spotify_id || node.itunes_id || node.deezer_id) { - setTimeout(() => openYourArtistInfoModal_direct(node), 200); - } + _artMapEmitRipple(node ? node.x : nx, node ? node.y : ny, node ? node._hue : null); + if (node && (node.spotify_id || node.itunes_id || node.deezer_id)) { + setTimeout(() => openYourArtistInfoModal_direct(node), 200); } } @@ -7001,6 +7017,7 @@ function _artMapSetupInteraction(canvas) { const wx = (t.clientX - rect.left - _artMap.offsetX) / _artMap.zoom; const wy = (t.clientY - rect.top - _artMap.offsetY) / _artMap.zoom; const node = _artMapHitTest(wx, wy); + _artMapEmitRipple(node ? node.x : wx, node ? node.y : wy, node ? node._hue : null); if (node && (node.spotify_id || node.itunes_id || node.deezer_id)) { openYourArtistInfoModal_direct(node); } From 682bdc7fb111b29cd801f5caa519683ee297c996 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 10:31:54 -0700 Subject: [PATCH 054/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20Phase?= =?UTF-8?q?=20F=20(perf):=20de-lag=20hover,=20dense=20zoom,=20fix=20toolti?= =?UTF-8?q?p=20photo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the perf + tooltip feedback: - Hover constellation no longer clips per node every frame (images are already pre-masked circles) — that per-node ctx.clip() was the hover-lag culprit once the ambient loop forced continuous redraws. Now a plain drawImage + arc tint. - Ambient buoyancy loop runs at ~30fps when idle (full 60 only during reveal/ripple), halving redraw cost on dense zoomed-in maps while keeping the bob smooth. - Gloss highlight gated to bubbles >=12px on screen (skips the dense swarm) — halves per-frame drawImage cost when zoomed in. - Tooltip photo now paints from the already-decoded bitmap into a canvas instead of a fresh reload — fixes the blank photo when sweeping across dense zoomed-in bubbles (the was churn-reloading faster than it could decode). 64 JS integrity tests pass. --- webui/static/discover.js | 50 +++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index b4b1e4d8..c3b0300a 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5533,8 +5533,9 @@ function _artMapDrawNodeToBuffer(octx, n, scale) { octx.fill(); } - // Glassy specular highlight (orb look) — skip the tiniest where it'd be noise. - if (rScaled >= 5) { + // Glassy specular highlight (orb look) — only on bubbles big enough to read + // it; skipping the dense swarm halves per-frame drawImage cost when zoomed in. + if (rScaled >= 12) { octx.drawImage(_artMapGlossSprite(), n.x - r, n.y - r, r * 2, r * 2); } @@ -5694,10 +5695,14 @@ function _artMapStartLoop() { if (!a.running) return; _artMap._now = t; const more = _artMapStepAnimations(t); - _artMapDraw(); // sets _artMap._liveCount - // Keep looping while something is animating, OR while ambient buoyancy is - // on AND there are live bubbles on screen to bob (so a zoomed-out static - // overview costs nothing — the loop parks until the next interaction). + // Reveal/ripple → full 60fps. Idle buoyancy → ~30fps (the gentle bob + // doesn't need 60, and halving the redraws keeps dense zoomed-in maps + // smooth + cool). + const ambientOnly = !more && _artMap._ambient; + if (!ambientOnly || (t - (a._lastDraw || 0)) >= 32) { + _artMapDraw(); // sets _artMap._liveCount + a._lastDraw = t; + } const keep = more || (_artMap._ambient && _artMap._liveCount > 0 && !document.hidden); if (keep) { a.raf = requestAnimationFrame(tick); @@ -5996,23 +6001,21 @@ function _artMapDraw() { ctx.stroke(); } - // Circle + image - ctx.save(); - ctx.beginPath(); - ctx.arc(hn.x, hn.y, r, 0, Math.PI * 2); - ctx.closePath(); - ctx.clip(); - + // Circle + image — pre-masked, so no per-frame clip (that was + // the hover-lag culprit once the ambient loop forced redraws). const img = _artMap.images[hn.id]; if (img) { ctx.drawImage(img, hn.x - r, hn.y - r, r * 2, r * 2); - ctx.fillStyle = 'rgba(0,0,0,0.35)'; - ctx.fillRect(hn.x - r, hn.y - r, r * 2, r * 2); + ctx.beginPath(); + ctx.arc(hn.x, hn.y, r, 0, Math.PI * 2); + ctx.fillStyle = 'rgba(0,0,0,0.35)'; // keep the name legible over art + ctx.fill(); } else { + ctx.beginPath(); + ctx.arc(hn.x, hn.y, r, 0, Math.PI * 2); ctx.fillStyle = isW ? '#1a0a30' : '#141420'; - ctx.fillRect(hn.x - r, hn.y - r, r * 2, r * 2); + ctx.fill(); } - ctx.restore(); // Border ctx.beginPath(); @@ -6155,7 +6158,13 @@ function _artMapShowTooltip(e, node) { // the hovered artist actually changes — a plain mousemove just repositions. if (_artMap._tipNodeId !== node.id) { _artMap._tipNodeId = node.id; - const img = node.image_url ? `` : '
♫
'; + // Prefer the already-decoded (pre-masked) bitmap — instant, and it can't + // churn-blank while sweeping across dense zoomed-in bubbles the way a + // fresh reload does. Fall back to the URL, then a glyph. + const bmp = _artMap.images[node.id]; + const img = bmp + ? '' + : (node.image_url ? `` : '
♫
'); const genres = (node.genres || []).slice(0, 3); const genreHTML = genres.length ? `
${genres.map(g => `${escapeHtml(g)}`).join('')}
` : ''; const typeLabel = node.type === 'watchlist' ? '★ Watchlist' : ''; @@ -6176,6 +6185,11 @@ function _artMapShowTooltip(e, node) {
`; tip.style.display = 'block'; + // Paint the cached bitmap into the tooltip canvas (instant, no reload). + if (bmp) { + const c = tip.querySelector('canvas.artmap-tip-img'); + if (c) { try { c.getContext('2d').drawImage(bmp, 0, 0, 88, 88); } catch (e) { /* ignore */ } } + } } // Position — keep on screen (cheap; runs every move) From 21b6d17d3c82ddb0fa70cea31771e734e965d732 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 10:46:41 -0700 Subject: [PATCH 055/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20Phase?= =?UTF-8?q?=20F=20(fix):=20robust=20live/buffer=20partition=20+=2030fps=20?= =?UTF-8?q?loop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the genre-map 'renders small/sparse after the reveal, zoom fixes it' bug. Root cause: tighter islands (Phase D) raised the fit-zoom so nearly every bubble crossed the live-size threshold → the buffer excluded them all (thought they were live) but the live layer is capped, so only ~600 of 1800 drew until a zoom rebuilt the partition. Fix: _artMapRebuildBuffer now counts would-be-live bubbles; if more than the live layer can draw (>450), it sets _liveOverflow and bakes EVERYTHING into the buffer (full, correct render). The live layer + bob only take over once zoomed in enough that few bubbles qualify. So the overview is always complete, regardless of zoom. Trade-off: very large maps (genre 1800) render from the buffer (no per-bubble bob, slightly softer when deeply zoomed until the zoom-rebuild sharpens) — correctness over flourish on the crowd. Also: whole animation loop capped at ~30fps (reveal/ripple/bob all read fine at 30) to cut the churn on dense maps; a pending rebuild (dirty) always draws so the throttle can't skip the post-reveal bake. 64 JS integrity tests pass. --- webui/static/discover.js | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index c3b0300a..732727c0 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5418,6 +5418,16 @@ function _artMapRebuildBuffer() { _artMap._liveBuildZoom = _artMap.zoom; _artMap._drawAlphaMul = 1; // buffer bakes at full alpha; the blit applies the reveal fade + // If more bubbles would be "live" than the live layer can draw, bake them ALL + // into the buffer (set the overflow flag BEFORE the draw loop so the live-size + // check below returns false and nothing is skipped). This is what prevents the + // genre-overview "small/sparse" render: the live layer caps out, so let the + // buffer own the whole crowd; live + bob only kick in once zoomed in. + const bz = _artMap.zoom; + let liveN = 0; + for (const n of visible) { if (!n._isLabel && (n.radius || 0) * bz >= _artMap.LIVE_PX) liveN++; } + _artMap._liveOverflow = liveN > 450; + octx.scale(scale, scale); octx.translate(-minX, -minY); @@ -5606,6 +5616,12 @@ function _artMapCompositeNode(n) { // so the two sets are always exact complements. function _artMapIsLiveSize(n) { if (n._isLabel) return false; + // When too many bubbles would be "live" at once (e.g. the genre overview), + // the live layer's cap can't draw them all and the buffer would exclude them + // → a sparse/half-rendered map. In that case treat NOTHING as live so the + // buffer bakes everything (full, correct render); the live layer + bob only + // take over once you've zoomed in to where few bubbles are big. + if (_artMap._liveOverflow) return false; const z = _artMap._liveBuildZoom || _artMap.zoom; return (n.radius || 0) * z >= _artMap.LIVE_PX; } @@ -5695,11 +5711,12 @@ function _artMapStartLoop() { if (!a.running) return; _artMap._now = t; const more = _artMapStepAnimations(t); - // Reveal/ripple → full 60fps. Idle buoyancy → ~30fps (the gentle bob - // doesn't need 60, and halving the redraws keeps dense zoomed-in maps - // smooth + cool). - const ambientOnly = !more && _artMap._ambient; - if (!ambientOnly || (t - (a._lastDraw || 0)) >= 32) { + // Cap the whole animation loop at ~30fps. The reveal bloom, ripples and + // ambient bob all read fine at 30, and halving the redraws keeps the + // 1800-bubble genre map smooth instead of churning every frame. Always + // honour a pending buffer rebuild (dirty) so the throttle can't skip the + // frame that bakes the map after the reveal ends. + if (_artMap.dirty || (t - (a._lastDraw || 0)) >= 31) { _artMapDraw(); // sets _artMap._liveCount a._lastDraw = t; } @@ -6840,9 +6857,10 @@ function _artMapSetupInteraction(canvas) { _artMap.zoom = newZoom; _artMapRender(); // fast blit _artMapEnsureAmbient(); // resume buoyancy if we zoomed bubbles into view - // Debounce hi-res rebuild after zoom settles + // Debounce hi-res rebuild after zoom settles; then resume buoyancy (the + // rebuild may have flipped the live/overflow partition). clearTimeout(_artMap._zoomRebuild); - _artMap._zoomRebuild = setTimeout(() => { _artMap.dirty = true; _artMapRender(); }, 300); + _artMap._zoomRebuild = setTimeout(() => { _artMap.dirty = true; _artMapRender(); _artMapEnsureAmbient(); }, 300); }, { passive: false }); let clickStart = null; From 80d775a5da8087025971fd4ebdfc311973316f36 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 11:00:10 -0700 Subject: [PATCH 056/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20one-isl?= =?UTF-8?q?and-at-a-time=20view=20+=20API-backed=20toolbar=20search?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things from feedback: 1) Toolbar search now queries the metadata source for ANY artist (like the discover page) and launches an exploration on click — instead of only filtering the current map's nodes (which showed nothing for off-map names). 2) Genre + watchlist maps now frame ONE genre island at a time, with prev/next nav (and ← / → keys) through the genres. This sidesteps the persistent 'renders small/sparse' bug entirely: only the focused island is visible, so the buffer covers a small region at HIGH res (crisp covers, no more shrunk images) and the live layer handles just ~hundreds of bubbles (bob works, no overflow). Each island blooms in (drop-in-water) on focus. Explore stays multi-island (it's small). A bottom nav bar shows genre name + i/N. Streaming caches off-island images silently (no redraw) so navigating is instant. 64 JS integrity tests pass. --- webui/static/discover.js | 172 ++++++++++++++++++++++++++++++++++----- 1 file changed, 152 insertions(+), 20 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index 732727c0..7f7ebead 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5215,6 +5215,98 @@ function _artMapFitToContent(marginPx = 120) { _artMap.offsetY = _artMap.height / 2 - ((minY + maxY) / 2) * _artMap.zoom; } +// ── One-island-at-a-time view (genre + watchlist) ───────────────────────── +// Big maps spread thousands of artists thin; fitting them all makes every cover +// tiny. Instead we frame ONE island filling the view (big crisp covers, live +// layer + bob viable on ~hundreds of bubbles) and navigate between them. Only +// the focused island is visible, so the buffer covers a small region at high +// res and the whole-world-buffer "small version" problem disappears. + +function _artMapFocusIsland(idx, opts = {}) { + const islands = _artMap._islands || []; + if (!islands.length) return; + idx = Math.max(0, Math.min(islands.length - 1, idx)); + _artMap._focusIdx = idx; + const isl = islands[idx]; + + // Show only this island's bubbles + its title; hide everything else so the + // buffer/zoom frame just this island. + for (const n of _artMap.placed) { + const mine = n._isLabel ? (n.name === isl.name) : (n._island === isl.name); + n.opacity = mine ? 1 : 0; + } + + // Frame the island in ~80% of the viewport. + const span = (isl.r * 2.3) + 120; + const z = Math.min(_artMap.width / span, _artMap.height / span, 1.2); + _artMap.zoom = z; + _artMap.offsetX = _artMap.width / 2 - isl.cx * z; + _artMap.offsetY = _artMap.height / 2 - isl.cy * z; + + _artMapUpdateIslandNav(); + + if (opts.bloom !== false) { + _artMapBloomIsland(isl); + } else { + _artMap.dirty = true; + _artMapRender(); + } +} + +// Bloom one island's bubbles (drop-in-water) + a ripple. Reuses the reveal loop. +function _artMapBloomIsland(isl) { + const t0 = performance.now(); + _artMap._revealing = true; + _artMap._ambient = true; + for (const n of _artMap.placed) { + if ((n.opacity || 0) < 0.01) continue; + n.aScale = 0; n.aAlpha = 0; + let radial = 0; + if (isl.r > 0) radial = Math.min(1, Math.hypot(n.x - isl.cx, n.y - isl.cy) / isl.r); + n._revealAt = t0 + (n._isLabel ? 60 : radial * 360); + n._revealDur = 430; + } + _artMap._ripples = [{ cx: isl.cx, cy: isl.cy, hue: isl.hue, maxR: isl.r * 1.4, t0, dur: 1000 }]; + _artMapStartLoop(); +} + +// Step to the prev/next island (wraps). +function _artMapIslandNav(dir) { + const islands = _artMap._islands || []; + if (islands.length < 2) return; + let idx = (_artMap._focusIdx || 0) + dir; + if (idx < 0) idx = islands.length - 1; + if (idx >= islands.length) idx = 0; + _artMapFocusIsland(idx, { bloom: true }); +} + +// Build/update the bottom nav bar (prev / genre name + i/N / next). Inline-styled +// so it doesn't depend on CSS that might not exist. +function _artMapUpdateIslandNav() { + const islands = _artMap._islands || []; + let nav = document.getElementById('artmap-island-nav'); + if (!_artMap._oneIsland || islands.length < 1) { if (nav) nav.remove(); return; } + const container = document.getElementById('artist-map-container'); + if (!container) return; + if (!nav) { + nav = document.createElement('div'); + nav.id = 'artmap-island-nav'; + nav.style.cssText = 'position:absolute;bottom:18px;left:50%;transform:translateX(-50%);display:flex;align-items:center;gap:14px;padding:8px 14px;background:rgba(16,12,28,0.82);backdrop-filter:blur(10px);border:1px solid rgba(168,85,247,0.25);border-radius:999px;z-index:30;box-shadow:0 6px 24px rgba(0,0,0,0.45);user-select:none;'; + container.appendChild(nav); + } + const idx = _artMap._focusIdx || 0; + const isl = islands[idx]; + const btn = 'width:30px;height:30px;border-radius:50%;border:1px solid rgba(255,255,255,0.18);background:rgba(255,255,255,0.06);color:#fff;font-size:13px;cursor:pointer;display:flex;align-items:center;justify-content:center;'; + nav.innerHTML = ` + +
+
${escapeHtml((isl.name || '').toUpperCase())}
+
${isl.count} artists · ${idx + 1} / ${islands.length}
+
+ + `; +} + async function openArtistMap() { const container = document.getElementById('artist-map-container'); if (!container) return; @@ -5269,7 +5361,7 @@ async function openArtistMap() { const groups = _artMapGroupByGenre(rawNodes); _artMapLayoutIslands(groups); _artMap.edges = _artMapRemapEdges(data.edges); - _artMapFitToContent(); + _artMap._oneIsland = true; // focus one genre island at a time // Setup interaction _artMapSetupInteraction(canvas); @@ -5299,7 +5391,7 @@ async function openArtistMap() { const le = document.getElementById('artist-map-loading'); if (le) le.remove(); - _artMapBeginReveal(); + _artMapFocusIsland(0, { bloom: true }); // frame + bloom the first genre island _artMapStreamImages(_artMap.placed); }, 50); @@ -5372,6 +5464,9 @@ function closeArtistMap() { _artMap._ambient = false; _artMap._anim.running = false; if (_artMap._anim.raf) { cancelAnimationFrame(_artMap._anim.raf); _artMap._anim.raf = null; } + _artMap._oneIsland = false; + const navEl = document.getElementById('artmap-island-nav'); + if (navEl) navEl.remove(); if (_artMap._keyHandler) window.removeEventListener('keydown', _artMap._keyHandler); _artMapHideContextMenu(); @@ -5426,7 +5521,10 @@ function _artMapRebuildBuffer() { const bz = _artMap.zoom; let liveN = 0; for (const n of visible) { if (!n._isLabel && (n.radius || 0) * bz >= _artMap.LIVE_PX) liveN++; } - _artMap._liveOverflow = liveN > 450; + // A single focused island (≤ maxPerIsland 500) should render via the crisp + // live layer; only fall back to baking-everything when there are clearly more + // bubbles than the live layer can draw. + _artMap._liveOverflow = liveN > 650; octx.scale(scale, scale); octx.translate(-minX, -minY); @@ -6129,23 +6227,51 @@ function _artMapDrawPerf(ctx, t0) { ctx.restore(); } +// Toolbar search: query the metadata source for ANY artist (like the discover +// page) and launch an exploration on click — not just filter the current map. function artMapSearch(query) { const results = document.getElementById('artist-map-search-results'); if (!results) return; - if (!query || query.length < 2) { results.style.display = 'none'; return; } + const q = (query || '').trim(); + if (q.length < 2) { results.style.display = 'none'; results.innerHTML = ''; return; } - const q = query.toLowerCase(); - const matches = _artMap.placed.filter(n => (n.opacity || 0) > 0.5 && n.name.toLowerCase().includes(q)).slice(0, 8); + clearTimeout(_artMap._searchTimer); + _artMap._searchTimer = setTimeout(async () => { + const myToken = (_artMap._searchToken = (_artMap._searchToken || 0) + 1); + results.style.display = 'block'; + results.innerHTML = '
Searching…
'; + try { + const resp = await fetch(`/api/discover/build-playlist/search-artists?query=${encodeURIComponent(q)}`); + const data = await resp.json(); + if (myToken !== _artMap._searchToken) return; // superseded by a newer keystroke + const artists = (data && data.success && Array.isArray(data.artists)) ? data.artists : []; + if (!artists.length) { + results.innerHTML = '
No artists found
'; + return; + } + results.innerHTML = artists.slice(0, 8).map(a => + `
+ ○ + ${escapeHtml(a.name)} + Explore → +
` + ).join(''); + } catch (e) { + if (myToken === _artMap._searchToken) { + results.innerHTML = '
Search failed — try again
'; + } + } + }, 300); +} - if (!matches.length) { results.style.display = 'none'; return; } - - results.style.display = 'block'; - results.innerHTML = matches.map(n => - `
- ${n.type === 'watchlist' ? '★' : '○'} - ${escapeHtml(n.name)} -
` - ).join(''); +// Launch the explorer for a searched artist (closes the dropdown first). +function artMapExploreArtist(name) { + const results = document.getElementById('artist-map-search-results'); + if (results) { results.style.display = 'none'; results.innerHTML = ''; } + const input = document.getElementById('artist-map-search'); + if (input) input.value = ''; + _artMap._skipSectionToggle = true; + _openArtistMapExplorerWithName(name); } function artMapZoomToNode(nodeId) { @@ -6461,7 +6587,7 @@ async function _openGenreMapWithSelection(selectedGenre) { })); _artMapLayoutIslands(groups); _artMap.edges = []; - _artMapFitToContent(); + _artMap._oneIsland = true; // focus one genre island at a time const placedCount = _artMap.placed.filter(n => !n._isLabel).length; _artMapSetupInteraction(canvas); @@ -6472,8 +6598,7 @@ async function _openGenreMapWithSelection(selectedGenre) { const le = document.getElementById('artist-map-loading'); if (le) le.remove(); - _artMap.dirty = true; - _artMapBeginReveal(); + _artMapFocusIsland(0, { bloom: true }); // frame + bloom the selected genre island // Stream images in throttled waves — interactive immediately, sharpens in place. _artMapStreamImages(_artMap.placed.filter(n => !n._isLabel)); @@ -6570,6 +6695,8 @@ async function _openArtistMapExplorerWithName(name) { const groups = _artMapGroupByGenre(rawNodes); _artMapLayoutIslands(groups); _artMap.edges = _artMapRemapEdges(data.edges); + _artMap._oneIsland = false; // explore stays multi-island (it's small) + _artMapUpdateIslandNav(); // tear down any leftover nav from a prior map _artMapFitToContent(); _artMapSetupInteraction(canvas); @@ -6813,10 +6940,13 @@ function _artMapStreamImages(imgNodes, concurrent = 24) { .then(bmp => { if (bmp && token === _artMap._loadToken) { _artMap.images[n.id] = bmp; + // Hidden bubbles (other islands in one-island mode): just + // cache the image for when you navigate there — don't + // redraw/rebuild for something off-screen. + if ((n.opacity || 0) < 0.01) return; // Composite just this node into the existing buffer (cheap) // and blit. Only fall back to a full rebuild if the buffer - // isn't built yet. This keeps pan/hover at blit-speed the - // whole time images are streaming in — the lag fix. + // isn't built yet. Keeps pan/hover at blit-speed while images stream. if (_artMapCompositeNode(n)) _artMapRender(); else scheduleRedraw(); } @@ -6885,6 +7015,8 @@ function _artMapSetupInteraction(canvas) { _artMap.dirty = true; _artMapRender(); } + else if (_artMap._oneIsland && e.key === 'ArrowLeft') { _artMapIslandNav(-1); e.preventDefault(); } + else if (_artMap._oneIsland && e.key === 'ArrowRight') { _artMapIslandNav(1); e.preventDefault(); } } window.addEventListener('keydown', _artMapKeyHandler); _artMap._keyHandler = _artMapKeyHandler; From e2eac64dce16bd0860f8a8586e94cfbe68602a41 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 11:09:02 -0700 Subject: [PATCH 057/108] Artist Map v2: de-lag one-island genre view + move island nav to top-left MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Focused islands now render from the high-res buffer (one cheap crisp blit) instead of redrawing every bubble each frame for the bob. In one-island mode the buffer already covers just that island at high resolution, so this is crisp AND cheap — kills the genre lag. Bob/shove stay live only for small views (zoomed-in subsets, explore) where per-frame redraw is cheap. (Overflow threshold 650→140; the loop parks once the island bakes.) - Fewer bubbles per island (maxPerIsland 500→300) — less cramped, lighter bloom. - Island nav bar moved from bottom-center to top-left (clears the genre sidebar + toolbar). 64 JS integrity tests pass. --- webui/static/discover.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index 7f7ebead..245d5c3c 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5122,7 +5122,7 @@ function _artMapLayoutIslands(groups, opts = {}) { _artMap._islands = []; const nodeR = opts.nodeR || (_artMap.WATCHLIST_R * 0.22); const gap = opts.gap || (_artMap.BUFFER * 2.2); - const cap = opts.maxPerIsland || 500; + const cap = opts.maxPerIsland || 300; let pid = 0; const islands = groups.map(g => { @@ -5291,9 +5291,12 @@ function _artMapUpdateIslandNav() { if (!nav) { nav = document.createElement('div'); nav.id = 'artmap-island-nav'; - nav.style.cssText = 'position:absolute;bottom:18px;left:50%;transform:translateX(-50%);display:flex;align-items:center;gap:14px;padding:8px 14px;background:rgba(16,12,28,0.82);backdrop-filter:blur(10px);border:1px solid rgba(168,85,247,0.25);border-radius:999px;z-index:30;box-shadow:0 6px 24px rgba(0,0,0,0.45);user-select:none;'; container.appendChild(nav); } + // Position top-left, clearing the genre sidebar (when shown) and the toolbar. + const sb = document.getElementById('artmap-genre-sidebar'); + const sbW = (sb && sb.style.display !== 'none') ? (sb.offsetWidth || 0) : 0; + nav.style.cssText = `position:absolute;top:64px;left:${sbW + 16}px;display:flex;align-items:center;gap:12px;padding:7px 12px;background:rgba(16,12,28,0.82);backdrop-filter:blur(10px);border:1px solid rgba(168,85,247,0.25);border-radius:14px;z-index:30;box-shadow:0 6px 24px rgba(0,0,0,0.45);user-select:none;`; const idx = _artMap._focusIdx || 0; const isl = islands[idx]; const btn = 'width:30px;height:30px;border-radius:50%;border:1px solid rgba(255,255,255,0.18);background:rgba(255,255,255,0.06);color:#fff;font-size:13px;cursor:pointer;display:flex;align-items:center;justify-content:center;'; @@ -5521,10 +5524,12 @@ function _artMapRebuildBuffer() { const bz = _artMap.zoom; let liveN = 0; for (const n of visible) { if (!n._isLabel && (n.radius || 0) * bz >= _artMap.LIVE_PX) liveN++; } - // A single focused island (≤ maxPerIsland 500) should render via the crisp - // live layer; only fall back to baking-everything when there are clearly more - // bubbles than the live layer can draw. - _artMap._liveOverflow = liveN > 650; + // In one-island mode the buffer already covers just the focused island at + // high resolution, so let the BUFFER own any non-trivial crowd (one cheap + // crisp blit, no per-frame redraw → no lag). The live layer + bob/shove only + // take over for small views (zoomed-in subsets, explore) where redrawing a + // handful of bubbles each frame is cheap. + _artMap._liveOverflow = liveN > 140; octx.scale(scale, scale); octx.translate(-minX, -minY); From 3e117909da12489d1c15d18438977372e63f4077 Mon Sep 17 00:00:00 2001 From: Arvuno Date: Wed, 3 Jun 2026 19:43:07 +0000 Subject: [PATCH 058/108] docs(readme): document AcoustID API key + retag gotcha The AcoustID section in the README was a one-liner that did not explain how to get the API key or why the retag action sometimes appears to do nothing. Expand the section with: - A short paragraph pointing users at acoustid.org/new-application and the Settings entry where the key belongs. - A note cross-referencing issue #704: the retag path treats an existing MUSICBRAINZ_TRACKID tag as a short-circuit, so removing the cached tag is the documented workaround. This gives a definitive answer until the retag path itself is fixed. No code or behavioural change. --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index a79eeb7e..325ee12b 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,21 @@ SoulSync bridges streaming services to your music library with automated discove - Catches wrong versions (live, remix, cover) even from streaming API sources - Fail-open design: verification errors never block downloads +#### AcoustID API key + +AcoustID verification is opt-in. To enable it, request a free API key +at and paste it into +Settings → AcoustID. Without a key, downloads still complete but the +verification step is skipped silently. + +If a track was previously tagged by AcoustID but the retag action in +the AcoustID Scanner no longer changes anything, see issue #704 — the +most common cause is that the file already carries a +`MUSICBRAINZ_TRACKID` tag, which the retag step uses as a short-circuit +and therefore never overwrites. Removing the cached +`MUSICBRAINZ_TRACKID` (and the `ACOUSTID_ID` if present) from the file +restores the retag. + ### Metadata & Enrichment **10 Background Enrichment Workers**: Spotify, MusicBrainz, iTunes, Deezer, Discogs, AudioDB, Last.fm, Genius, Tidal, Qobuz From f95b52e3c42d879313d09102fe0aa5fddbb18c0d Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 13:01:58 -0700 Subject: [PATCH 059/108] =?UTF-8?q?Artist=20Map=20v2=20=E2=80=94=20polish:?= =?UTF-8?q?=20island=20halo,=20hover-pop,=20genre=20quick-jump,=20declutte?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Soft genre-hued halo glows behind the focused island (cached per-hue sprite → one drawImage, no per-frame gradient) so it reads as a place on the water. - Hover-pop: hovered bubbles scale up + get a bright hue ring + glow, even on static genre islands (drawn on top), so hover always feels tactile/responsive. - Genre quick-jump: click the genre name in the nav for a dropdown of every genre island — jump straight to one instead of only prev/next. - Decluttered: dropped the redundant in-world island titles in one-island mode (the nav bar already names the genre, and they could clip off the top). 64 JS integrity tests pass. --- webui/static/discover.js | 127 +++++++++++++++++++++++++++++++++------ 1 file changed, 107 insertions(+), 20 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index 245d5c3c..dad9bad8 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5059,6 +5059,26 @@ function _artMapGlossSprite() { return c; } +// A cached soft radial "halo" sprite per genre hue — drawn behind the focused +// island so it reads as a glowing place on the water. Cached per hue (≤ a few), +// so it's just a drawImage per frame, never a per-frame gradient. +function _artMapHaloSprite(hue) { + _artMap._halos = _artMap._halos || {}; + if (_artMap._halos[hue]) return _artMap._halos[hue]; + const S = 256; + const c = document.createElement('canvas'); + c.width = S; c.height = S; + const cx = c.getContext('2d'); + const g = cx.createRadialGradient(S / 2, S / 2, 0, S / 2, S / 2, S / 2); + g.addColorStop(0, `hsla(${hue},75%,55%,0.22)`); + g.addColorStop(0.45, `hsla(${hue},75%,50%,0.08)`); + g.addColorStop(1, `hsla(${hue},75%,50%,0)`); + cx.fillStyle = g; + cx.fillRect(0, 0, S, S); + _artMap._halos[hue] = c; + return c; +} + // Deterministic hue (0–360) from a genre name, so each island has a stable tint. function _artMapGenreHue(name) { let h = 0; @@ -5229,11 +5249,11 @@ function _artMapFocusIsland(idx, opts = {}) { _artMap._focusIdx = idx; const isl = islands[idx]; - // Show only this island's bubbles + its title; hide everything else so the - // buffer/zoom frame just this island. + // Show only this island's bubbles; hide everything else (and the in-world + // titles — the nav bar already names the genre) so the frame is just this + // island's covers. for (const n of _artMap.placed) { - const mine = n._isLabel ? (n.name === isl.name) : (n._island === isl.name); - n.opacity = mine ? 1 : 0; + n.opacity = (!n._isLabel && n._island === isl.name) ? 1 : 0; } // Frame the island in ~80% of the viewport. @@ -5299,15 +5319,53 @@ function _artMapUpdateIslandNav() { nav.style.cssText = `position:absolute;top:64px;left:${sbW + 16}px;display:flex;align-items:center;gap:12px;padding:7px 12px;background:rgba(16,12,28,0.82);backdrop-filter:blur(10px);border:1px solid rgba(168,85,247,0.25);border-radius:14px;z-index:30;box-shadow:0 6px 24px rgba(0,0,0,0.45);user-select:none;`; const idx = _artMap._focusIdx || 0; const isl = islands[idx]; - const btn = 'width:30px;height:30px;border-radius:50%;border:1px solid rgba(255,255,255,0.18);background:rgba(255,255,255,0.06);color:#fff;font-size:13px;cursor:pointer;display:flex;align-items:center;justify-content:center;'; + const btn = 'width:30px;height:30px;border-radius:50%;border:1px solid rgba(255,255,255,0.18);background:rgba(255,255,255,0.06);color:#fff;font-size:13px;cursor:pointer;display:flex;align-items:center;justify-content:center;flex:none;'; nav.innerHTML = ` -
-
${escapeHtml((isl.name || '').toUpperCase())}
+
+
${escapeHtml((isl.name || '').toUpperCase())} ▾
${isl.count} artists · ${idx + 1} / ${islands.length}
`; + // Re-anchor an open menu (or leave it closed). + const menu = document.getElementById('artmap-island-menu'); + if (menu) menu.remove(); +} + +// Quick-jump dropdown: list every genre island; click to jump straight to it. +function _artMapToggleIslandMenu(ev) { + if (ev) ev.stopPropagation(); + const existing = document.getElementById('artmap-island-menu'); + if (existing) { existing.remove(); return; } + const islands = _artMap._islands || []; + const nav = document.getElementById('artmap-island-nav'); + if (!nav || !islands.length) return; + const menu = document.createElement('div'); + menu.id = 'artmap-island-menu'; + menu.style.cssText = `position:absolute;top:${nav.offsetTop + nav.offsetHeight + 6}px;left:${nav.offsetLeft}px;min-width:${Math.max(180, nav.offsetWidth)}px;max-height:50vh;overflow-y:auto;background:rgba(16,12,28,0.96);backdrop-filter:blur(10px);border:1px solid rgba(168,85,247,0.25);border-radius:12px;z-index:31;box-shadow:0 8px 28px rgba(0,0,0,0.5);padding:6px;`; + const cur = _artMap._focusIdx || 0; + menu.innerHTML = islands.map((isl, i) => ` +
+ + + ${escapeHtml(isl.name)} + + ${isl.count} +
`).join(''); + nav.parentElement.appendChild(menu); + // Close on next outside click. + setTimeout(() => { + const close = (e) => { if (!menu.contains(e.target)) { menu.remove(); document.removeEventListener('mousedown', close); } }; + document.addEventListener('mousedown', close); + }, 0); +} + +function _artMapJumpIsland(i) { + const menu = document.getElementById('artmap-island-menu'); + if (menu) menu.remove(); + _artMapFocusIsland(i, { bloom: true }); } async function openArtistMap() { @@ -5770,6 +5828,32 @@ function _artMapDrawLiveLayer(ctx) { _artMap._liveCount = revealing ? 0 : drawn; } +// Tactile hover-pop: redraw the hovered bubble slightly larger with its cover +// + a bright hue ring, on top of everything. Works even when the bubble lives +// in the static buffer (genre islands), so hover always feels responsive. +// ctx is already in world space (translate(offset) + scale(zoom)). +function _artMapDrawHoverPop(ctx, n) { + const r = n.radius; + const hue = n._hue == null ? 270 : n._hue; + const s = 1.16; + const img = _artMap.images[n.id]; + ctx.save(); + ctx.translate(n.x, n.y); ctx.scale(s, s); ctx.translate(-n.x, -n.y); + if (img) { + ctx.drawImage(img, n.x - r, n.y - r, r * 2, r * 2); + } else { + ctx.beginPath(); ctx.arc(n.x, n.y, r, 0, Math.PI * 2); + ctx.fillStyle = '#1a0a30'; ctx.fill(); + } + ctx.beginPath(); ctx.arc(n.x, n.y, r, 0, Math.PI * 2); + ctx.strokeStyle = `hsla(${hue},90%,78%,0.95)`; + ctx.lineWidth = 2.5 / s; ctx.stroke(); + ctx.restore(); + ctx.beginPath(); ctx.arc(n.x, n.y, r * s + 5, 0, Math.PI * 2); + ctx.strokeStyle = `hsla(${hue},85%,66%,0.45)`; + ctx.lineWidth = 3; ctx.stroke(); +} + // Draw one live bubble with its animation transform. aScale scales about the // node centre; aAlpha fades it (folded into the global draw-alpha multiplier). // Reuses the shared node painter so the bubble is identical to its baked form @@ -6008,6 +6092,18 @@ function _artMapDraw() { const z = _artMap.zoom; + // Soft genre-hued halo behind the focused island (one-island mode) — gives + // the island a sense of place on the water. Cached sprite → one drawImage. + if (_artMap._oneIsland && _artMap._islands && _artMap._islands.length) { + const isl = _artMap._islands[_artMap._focusIdx || 0]; + if (isl) { + const hr = (isl.r * 2.5) * z; + const hsx = _artMap.offsetX + isl.cx * z; + const hsy = _artMap.offsetY + isl.cy * z; + ctx.drawImage(_artMapHaloSprite(isl.hue), hsx - hr, hsy - hr, hr * 2, hr * 2); + } + } + // While the ripple-bloom reveal is running, bypass the static buffer // entirely and let the live layer draw every bubble (so each can animate). // The buffer is (re)built once when the reveal ends. @@ -6156,27 +6252,18 @@ function _artMapDraw() { } ctx.globalAlpha = 1; } else { - // Single node, no connections - ctx.beginPath(); - ctx.arc(n.x, n.y, n.radius + 4, 0, Math.PI * 2); - ctx.strokeStyle = 'rgba(255,255,255,0.5)'; - ctx.lineWidth = 3; - ctx.stroke(); + // Single node, no connections — pop the hovered bubble. + _artMapDrawHoverPop(ctx, n); } ctx.restore(); } // end if(n) } else if (_artMap.hoveredNode && !_artMap._constellationActive) { - // Pre-constellation: just show a simple highlight ring (instant, no delay) - const n = _artMap.hoveredNode; + // Pre-constellation: instant tactile pop on the hovered bubble. ctx.save(); ctx.translate(_artMap.offsetX, _artMap.offsetY); ctx.scale(z, z); - ctx.beginPath(); - ctx.arc(n.x, n.y, n.radius + 3, 0, Math.PI * 2); - ctx.strokeStyle = 'rgba(255,255,255,0.35)'; - ctx.lineWidth = 2; - ctx.stroke(); + _artMapDrawHoverPop(ctx, _artMap.hoveredNode); ctx.restore(); } From 37595314f153a3703aac15d4b5dfb0c934cf1739 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 13:13:09 -0700 Subject: [PATCH 060/108] Artist Map v2 fix: streamed art now appears on its own (no manual zoom) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the 'loads as placeholder orbs, only pops in after a zoom' bug: streamed images were cached in _artMap.images but written into the buffer via the per-node composite path, which didn't reliably refresh in one-island / overflow mode — so covers stayed as placeholders until a zoom forced a full rebuild that picked up the cached bitmaps. Now that each map's buffer is small (one focused island, or a small explore map), a throttled FULL rebuild on image arrival is cheap and always bakes every cached image. Dropped the composite call from the stream; art fills in by itself as it loads. 64 JS integrity tests pass. --- webui/static/discover.js | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index dad9bad8..d6c3a3e0 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -7009,8 +7009,11 @@ function _artMapStreamImages(imgNodes, concurrent = 24) { const queue = imgNodes.filter(n => n.image_url).slice().sort((a, b) => (b.radius || 0) - (a.radius || 0)); let idx = 0, inFlight = 0, redrawPending = false; - // Fallback full-rebuild path, used only if the buffer isn't ready yet - // (e.g. images returned before the first paint built the offscreen buffer). + // Throttled FULL rebuild as images arrive. The per-map buffer is now small + // (one focused island / a small explore map), so a full rebuild is cheap and + // — unlike the per-node composite — is guaranteed to pick up every cached + // image. This is what makes streamed art appear on its own instead of only + // after a manual zoom forced a rebuild. const scheduleRedraw = () => { if (redrawPending || token !== _artMap._loadToken) return; redrawPending = true; @@ -7019,7 +7022,8 @@ function _artMapStreamImages(imgNodes, concurrent = 24) { if (token !== _artMap._loadToken) return; _artMap.dirty = true; _artMapRender(); - }, 280); + _artMapEnsureAmbient(); + }, 200); }; function pump() { @@ -7034,13 +7038,11 @@ function _artMapStreamImages(imgNodes, concurrent = 24) { _artMap.images[n.id] = bmp; // Hidden bubbles (other islands in one-island mode): just // cache the image for when you navigate there — don't - // redraw/rebuild for something off-screen. + // redraw for something off-screen. if ((n.opacity || 0) < 0.01) return; - // Composite just this node into the existing buffer (cheap) - // and blit. Only fall back to a full rebuild if the buffer - // isn't built yet. Keeps pan/hover at blit-speed while images stream. - if (_artMapCompositeNode(n)) _artMapRender(); - else scheduleRedraw(); + // Throttled full rebuild — reliably bakes newly-arrived art + // into the (small) buffer. No manual zoom needed. + scheduleRedraw(); } }) .finally(() => { inFlight--; pump(); }); From 0d91bb78db91314407e08641898b91891c814e62 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 13:19:25 -0700 Subject: [PATCH 061/108] =?UTF-8?q?Artist=20Map=20v2:=20fancier=20bloom=20?= =?UTF-8?q?=E2=80=94=20bubbles=20surface=20up=20into=20place?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bubbles now rise up into position (water-surfacing) with a soft ease-out-back settle and alpha fading in a touch faster than scale. Stagger is continuous radial + a deterministic per-bubble jitter so they fill in organically instead of popping in visible rings/segments. 64 JS integrity tests pass. --- webui/static/discover.js | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index d6c3a3e0..42e4b8aa 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5283,10 +5283,15 @@ function _artMapBloomIsland(isl) { n.aScale = 0; n.aAlpha = 0; let radial = 0; if (isl.r > 0) radial = Math.min(1, Math.hypot(n.x - isl.cx, n.y - isl.cy) / isl.r); - n._revealAt = t0 + (n._isLabel ? 60 : radial * 360); - n._revealDur = 430; + // Continuous radial stagger + deterministic per-bubble jitter so they + // surface organically rather than in visible rings/segments. + const jitter = ((Math.abs((n.id | 0) * 1103515245 + 12345) % 1000) / 1000) * 200; + n._revealAt = t0 + radial * 300 + jitter; + n._revealDur = 560; + n._riseAmp = (n.radius || 20) * 1.15; // bubbles rise up into place (surfacing) + n._revealRise = n._riseAmp; } - _artMap._ripples = [{ cx: isl.cx, cy: isl.cy, hue: isl.hue, maxR: isl.r * 1.4, t0, dur: 1000 }]; + _artMap._ripples = [{ cx: isl.cx, cy: isl.cy, hue: isl.hue, maxR: isl.r * 1.45, t0, dur: 1100 }]; _artMapStartLoop(); } @@ -5866,7 +5871,9 @@ function _artMapDrawLiveNode(ctx, n) { // Ambient buoyancy + ripple shove (steady state only — the reveal has its // own motion). Both are world-space offsets applied about the node centre. let ox = 0, oy = 0; - if (!_artMap._revealing) { + if (_artMap._revealing) { + if (n._revealRise) oy += n._revealRise; // surfacing rise during the bloom + } else { if (n._bobAmp) oy += Math.sin((_artMap._now || 0) * 0.0016 + (n._bobPhase || 0)) * n._bobAmp; const disp = _artMapNodeDisplacement(n); if (disp) { ox += disp.dx; oy += disp.dy; } @@ -5936,9 +5943,19 @@ function _artMapStepAnimations(t) { if (n.aScale == null || n.aScale >= 1) continue; if (t < n._revealAt) { active = true; continue; } const p = Math.min(1, (t - n._revealAt) / (n._revealDur || 480)); - const e = 1 - Math.pow(1 - p, 3); // ease-out-cubic - if (p >= 1) { n.aScale = 1; n.aAlpha = 1; } - else { n.aScale = 0.55 + 0.45 * e; n.aAlpha = e; active = true; } + if (p >= 1) { n.aScale = 1; n.aAlpha = 1; n._revealRise = 0; } + else { + // Scale eases in with a gentle overshoot (ease-out-back, subtle), + // alpha fades a touch faster, and the bubble rises up into place + // like it's surfacing through water — the remaining rise decays + // as (1-p)^3. + const c1 = 1.18, c3 = c1 + 1; + const back = 1 + c3 * Math.pow(p - 1, 3) + c1 * Math.pow(p - 1, 2); + n.aScale = back; + n.aAlpha = Math.min(1, p * 1.6); + n._revealRise = Math.pow(1 - p, 3) * (n._riseAmp || 0); + active = true; + } } } From b0c0acb379225470ef029309629281a2aaae8a3f Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 13:28:57 -0700 Subject: [PATCH 062/108] Artist Map v2: right-side info panel (dashboard + coverage + top artists + artist card) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A polished detail panel on the right of every map (never collides with the genre sidebar; islands now frame in the space left of it): - Header dashboard: view title, Artists / Watchlist / Genres stat tiles, and a watchlist-coverage bar for the current genre/view. - Top-artists list: the current island's biggest artists, clickable (shows their card + ripples them on the map). - Rich artist card on hover/click: large art (from the decoded bitmap), genre chips, popularity bar, connection count, watchlist/discovered badge, and actions — Explore from here, Details, Watchlist toggle, Open artist page. Card stays pinned (no auto-revert) so you can reach its buttons; a back button returns to the list. 64 JS integrity tests pass. --- webui/static/discover.js | 186 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 181 insertions(+), 5 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index 42e4b8aa..e85404bc 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5025,6 +5025,7 @@ const _artMap = { _anim: { running: false, raf: null, last: 0 }, _fieldAlpha: 1, // global fade for the static far-field buffer (reveal) _revealT0: 0, // performance.now() when the current reveal began + _panelW: 320, // right-side info panel width (reserved when framing islands) }; // ── Genre-island layout (shared by watchlist / genre / explore) ─────────── @@ -5229,9 +5230,10 @@ function _artMapFitToContent(marginPx = 120) { minY = Math.min(minY, n.y - n.radius); maxY = Math.max(maxY, n.y + n.radius); } if (!isFinite(minX)) return; + const usableW = Math.max(200, _artMap.width - _artMap._panelW); const mapW = maxX - minX + marginPx * 2, mapH = maxY - minY + marginPx * 2; - _artMap.zoom = Math.min(_artMap.width / mapW, _artMap.height / mapH, 1); - _artMap.offsetX = _artMap.width / 2 - ((minX + maxX) / 2) * _artMap.zoom; + _artMap.zoom = Math.min(usableW / mapW, _artMap.height / mapH, 1); + _artMap.offsetX = usableW / 2 - ((minX + maxX) / 2) * _artMap.zoom; _artMap.offsetY = _artMap.height / 2 - ((minY + maxY) / 2) * _artMap.zoom; } @@ -5256,14 +5258,16 @@ function _artMapFocusIsland(idx, opts = {}) { n.opacity = (!n._isLabel && n._island === isl.name) ? 1 : 0; } - // Frame the island in ~80% of the viewport. + // Frame the island in the space LEFT of the info panel (~80% of it). + const usableW = Math.max(200, _artMap.width - _artMap._panelW); const span = (isl.r * 2.3) + 120; - const z = Math.min(_artMap.width / span, _artMap.height / span, 1.2); + const z = Math.min(usableW / span, _artMap.height / span, 1.2); _artMap.zoom = z; - _artMap.offsetX = _artMap.width / 2 - isl.cx * z; + _artMap.offsetX = usableW / 2 - isl.cx * z; _artMap.offsetY = _artMap.height / 2 - isl.cy * z; _artMapUpdateIslandNav(); + _artMapRefreshPanel(); if (opts.bloom !== false) { _artMapBloomIsland(isl); @@ -5373,6 +5377,172 @@ function _artMapJumpIsland(i) { _artMapFocusIsland(i, { bloom: true }); } +// ── Right-side info panel ────────────────────────────────────────────────── +// A polished detail panel: a discovery dashboard + current-view coverage at the +// top, a clickable top-artists list, and a rich artist card when you hover/click +// a bubble. Lives on the right so it never collides with the genre sidebar. + +function _artMapNodeBest(n) { + const map = [['spotify_id', 'spotify'], ['itunes_id', 'itunes'], ['deezer_id', 'deezer'], ['discogs_id', 'discogs'], ['musicbrainz_id', 'musicbrainz']]; + for (const [k, s] of map) if (n && n[k]) return { id: n[k], source: s }; + return { id: '', source: '' }; +} + +function _artMapConnCount(n) { + let c = 0; + for (const e of (_artMap.edges || [])) if (e.source === n.id || e.target === n.id) c++; + return c; +} + +function _artMapEnsurePanel() { + const container = document.getElementById('artist-map-container'); + if (!container) return null; + let p = document.getElementById('artmap-info-panel'); + if (!p) { + p = document.createElement('div'); + p.id = 'artmap-info-panel'; + p.style.cssText = `position:absolute;top:0;right:0;width:${_artMap._panelW}px;height:100%;` + + `background:linear-gradient(180deg,rgba(20,15,34,0.92),rgba(11,8,20,0.96));backdrop-filter:blur(16px);` + + `border-left:1px solid rgba(168,85,247,0.18);z-index:26;display:flex;flex-direction:column;` + + `color:#fff;overflow:hidden;box-shadow:-10px 0 36px rgba(0,0,0,0.45);font-size:13px;`; + p.innerHTML = `
` + + `
`; + container.appendChild(p); + } + return p; +} + +function _artMapClosePanel() { + const p = document.getElementById('artmap-info-panel'); + if (p) p.remove(); + _artMap._panelArtistId = null; +} + +// Stat tile helper +function _miniStat(label, value, hue) { + return `
+
${value}
+
${label}
+
`; +} + +// Build the panel header (dashboard) + body (top-artists list) for the view. +function _artMapRefreshPanel() { + const p = _artMapEnsurePanel(); + if (!p) return; + const head = document.getElementById('artmap-panel-head'); + const body = document.getElementById('artmap-panel-body'); + if (!head || !body) return; + + const nodes = (_artMap.placed || []).filter(n => !n._isLabel); + const total = nodes.length; + const watch = nodes.filter(n => n.type === 'watchlist' || n.type === 'center').length; + const islands = _artMap._islands || []; + const oneIsland = _artMap._oneIsland; + const isl = oneIsland && islands.length ? islands[_artMap._focusIdx || 0] : null; + + // Scope (current island vs whole map) + const scope = isl ? nodes.filter(n => n._island === isl.name) : nodes; + const scopeTotal = isl ? isl.count : total; + const scopeWatch = scope.filter(n => n.type === 'watchlist' || n.type === 'center').length; + const cov = scopeTotal ? Math.round((scopeWatch / scopeTotal) * 100) : 0; + const hue = isl ? isl.hue : 270; + + const title = _artMap._mapTitle || 'Artist Map'; + head.innerHTML = ` +
${title}
+ ${isl ? `
${escapeHtml(isl.name)}
` : `
Overview
`} +
+ ${_miniStat('Artists', scopeTotal, hue)} + ${_miniStat('Watchlist', scopeWatch)} + ${_miniStat(isl ? 'Genre' : 'Genres', isl ? '1' : (islands.length || 1))} +
+
+
+ On your watchlist${scopeWatch}/${scopeTotal} +
+
+
+
+
`; + + // Body: top artists (by popularity) in the current scope. + _artMap._panelArtistId = null; + const top = scope.slice().sort((a, b) => (b.popularity || 0) - (a.popularity || 0)).slice(0, 14); + body.innerHTML = ` +
Top artists
+ ${top.map((n, i) => ` +
+ ${i + 1} + + ${n.image_url ? `` : '♫'} + + ${escapeHtml(n.name)} + ${n.type === 'watchlist' ? '★' : ''} +
`).join('') || '
No artists
'}`; +} + +// Show a rich artist card in the body (hover/click); pass null to restore the list. +function _artMapPanelArtist(node) { + const body = document.getElementById('artmap-panel-body'); + if (!body) return; + if (!node) { if (_artMap._panelArtistId != null) _artMapRefreshPanel(); return; } + if (_artMap._panelArtistId === node.id) return; // already showing + _artMap._panelArtistId = node.id; + + const hue = node._hue == null ? 270 : node._hue; + const conn = _artMapConnCount(node); + const pop = Math.max(0, Math.min(100, Math.round(node.popularity || 0))); + const genres = (node.genres || []).slice(0, 5); + const best = _artMapNodeBest(node); + const typeLabel = (node.type === 'watchlist' || node.type === 'center') ? 'On watchlist' : 'Discovered'; + const bmp = _artMap.images[node.id]; + + body.innerHTML = ` + +
+
+ ${bmp ? `` + : (node.image_url ? `` : '♫')} +
+
${escapeHtml(node.name)}
+
${typeLabel}
+
+ ${genres.length ? `
+ ${genres.map(g => `${escapeHtml(g)}`).join('')} +
` : ''} +
+ ${_miniStat('Popularity', pop, hue)} + ${_miniStat('Connections', conn)} +
+
+
+
+
+ +
+ + +
+ ${best.id ? `Open artist page` : ''} +
`; + + if (bmp) { + const c = document.getElementById('artmap-card-canvas'); + if (c) { try { c.getContext('2d').drawImage(bmp, 0, 0, 120, 120); } catch (e) { /* ignore */ } } + } +} + +// Top-list / external entry: show a node's card by id (also ripples it on the map). +function _artMapPanelArtistById(id) { + const n = (_artMap._nodeById || {})[id]; + if (!n) return; + _artMapPanelArtist(n); + _artMapEmitRipple(n.x, n.y, n._hue); +} + async function openArtistMap() { const container = document.getElementById('artist-map-container'); if (!container) return; @@ -5428,6 +5598,7 @@ async function openArtistMap() { _artMapLayoutIslands(groups); _artMap.edges = _artMapRemapEdges(data.edges); _artMap._oneIsland = true; // focus one genre island at a time + _artMap._mapTitle = 'Watchlist Map'; // Setup interaction _artMapSetupInteraction(canvas); @@ -5533,6 +5704,7 @@ function closeArtistMap() { _artMap._oneIsland = false; const navEl = document.getElementById('artmap-island-nav'); if (navEl) navEl.remove(); + _artMapClosePanel(); if (_artMap._keyHandler) window.removeEventListener('keydown', _artMap._keyHandler); _artMapHideContextMenu(); @@ -6697,6 +6869,7 @@ async function _openGenreMapWithSelection(selectedGenre) { _artMapLayoutIslands(groups); _artMap.edges = []; _artMap._oneIsland = true; // focus one genre island at a time + _artMap._mapTitle = 'Genre Map'; const placedCount = _artMap.placed.filter(n => !n._isLabel).length; _artMapSetupInteraction(canvas); @@ -6805,8 +6978,10 @@ async function _openArtistMapExplorerWithName(name) { _artMapLayoutIslands(groups); _artMap.edges = _artMapRemapEdges(data.edges); _artMap._oneIsland = false; // explore stays multi-island (it's small) + _artMap._mapTitle = 'Explore: ' + (data.center || name); _artMapUpdateIslandNav(); // tear down any leftover nav from a prior map _artMapFitToContent(); + _artMapRefreshPanel(); _artMapSetupInteraction(canvas); @@ -7194,6 +7369,7 @@ function _artMapSetupInteraction(canvas) { _artMap.hoveredNode = _artMapHitTest(nx, ny); canvas.style.cursor = _artMap.hoveredNode ? 'pointer' : 'grab'; _artMapShowTooltip(e, _artMap.hoveredNode); + if (_artMap.hoveredNode) _artMapPanelArtist(_artMap.hoveredNode); // rich card in side panel if (prev !== _artMap.hoveredNode) { // Reset constellation highlight timer clearTimeout(_artMap._constellationTimer); From ed2a97d7010319b7b96c22a70dcf911ae4e3565b Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 13:40:00 -0700 Subject: [PATCH 063/108] Fix: artist detail forced Enhanced view on source-only artists, hiding discog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The persisted Standard/Enhanced preference was re-applied on every artist load BEFORE the data came back — so for an artist not in the library (source-only, no Enhanced view) it still flipped to Enhanced, which showed an empty Enhanced pane and never rendered the discography. Now the preference is applied inside loadArtistDetailData, after we know the artist's status (data.artist.server_source). Only library artists honour a saved 'enhanced' choice; source-only artists always stay on Standard (discography). --- webui/static/library.js | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/webui/static/library.js b/webui/static/library.js index 2eae1a2c..e9850567 100644 --- a/webui/static/library.js +++ b/webui/static/library.js @@ -879,15 +879,6 @@ function navigateToArtistDetail(artistId, artistName, sourceOverride = null, opt const bulkBar = document.getElementById('enhanced-bulk-bar'); if (bulkBar) bulkBar.classList.remove('visible'); - // Restore persisted view preference. Non-admins can't see / toggle the - // Enhanced control so only honour the saved choice for admins; default - // is still Standard. Wrapped in try/catch because localStorage can be - // disabled in private-browsing modes. - let _preferEnhanced = false; - try { - _preferEnhanced = localStorage.getItem(_libraryViewModeKey()) === 'enhanced'; - } catch (_) { /* localStorage unavailable */ } - // Navigate to artist detail page navigateToPage('artist-detail', { artistId, @@ -902,15 +893,11 @@ function navigateToArtistDetail(artistId, artistName, sourceOverride = null, opt _updateArtistDetailBackButtonLabel(); - // Load artist data + // Load artist data. The persisted Enhanced-view preference is applied INSIDE + // loadArtistDetailData, once we know whether this artist is in the library — + // source-only artists have no Enhanced view, so forcing it there left the + // Enhanced pane empty and hid the discography. loadArtistDetailData(artistId, artistName); - - // Apply persisted Enhanced view after the standard data load is kicked off. - // toggleEnhancedView() triggers its own loadEnhancedViewData() in parallel; - // the brief Standard render is hidden as soon as the toggle flips. - if (_preferEnhanced && isEnhancedAdmin()) { - toggleEnhancedView(true); - } } function _updateArtistDetailBackButtonLabel() { @@ -1095,6 +1082,18 @@ async function loadArtistDetailData(artistId, artistName) { checkArtistEnhanceEligibility(artistDetailPageState.currentArtistId); } + // Apply the persisted Enhanced/Standard preference now that we know the + // artist's status. Only LIBRARY artists have an Enhanced view — forcing + // it on a source-only artist (no DB record) showed an empty Enhanced pane + // and hid the discography. Source-only artists always stay on Standard. + if (!isSourceOnlyArtist && isEnhancedAdmin()) { + let _preferEnhanced = false; + try { + _preferEnhanced = localStorage.getItem(_libraryViewModeKey()) === 'enhanced'; + } catch (_) { /* localStorage unavailable */ } + if (_preferEnhanced) toggleEnhancedView(true); + } + } catch (error) { console.error(`❌ Error loading artist detail data:`, error); From 409595974e8c024574c2f1fb1e6259c29f95ff28 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 13:49:56 -0700 Subject: [PATCH 064/108] =?UTF-8?q?Artist=20Map=20v2:=20bring=20the=20info?= =?UTF-8?q?=20panel=20to=20life=20=E2=80=94=20live=20watchlist=20state,=20?= =?UTF-8?q?debounced=20select,=20navbar=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Watchlist button now reflects real state: shows 'On watchlist' (filled) vs 'Watchlist' (outline), confirmed per-artist via /api/watchlist/check, and flips instantly when you add/remove (cached in _watchSet so it stays correct as you browse). Uses the artist's source id, works on any map. - Debounced hover-select: the card only swaps to a bubble you've settled on for ~0.8s, so sweeping toward the panel no longer keeps changing the card on bubbles you pass over. Clicking a bubble selects it instantly (bypasses the debounce, pins the card) instead of auto-opening the modal — Details button still opens it. - Fix: panel started at top:0 and covered the navbar; now it starts below the .artist-map-toolbar (measured) so the toolbar stays clear. 64 tests pass. --- webui/static/discover.js | 108 +++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 10 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index e85404bc..b601274f 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5394,6 +5394,73 @@ function _artMapConnCount(n) { return c; } +// ── Watchlist state for the panel card ── +function _artMapIsWatched(n) { + if (!n) return false; + if (n.type === 'watchlist') return true; + const best = _artMapNodeBest(n); + return !!(best.id && _artMap._watchSet && _artMap._watchSet.has(best.id)); +} + +// The watchlist button's markup, reflecting current state (filled star + "On +// watchlist" when watched, outline + "Watchlist" when not). +function _artMapWatchBtnHtml(n) { + const watched = _artMapIsWatched(n); + const idArg = typeof n.id === 'number' ? n.id : `'${n.id}'`; + return ``; +} + +function _artMapRenderWatchBtn(n) { + const b = document.getElementById('artmap-card-watch'); + if (b) b.outerHTML = _artMapWatchBtnHtml(n); // inline onclick survives outerHTML swap +} + +// Toggle watchlist membership for a node, updating the cache + button in place. +async function _artMapToggleWatch(id) { + const n = (_artMap._nodeById || {})[id]; + if (!n) return; + const best = _artMapNodeBest(n); + if (!best.id) { showToast('No source id for this artist', 'error'); return; } + _artMap._watchSet = _artMap._watchSet || new Set(); + const watched = _artMapIsWatched(n); + try { + const resp = await fetch(watched ? '/api/watchlist/remove' : '/api/watchlist/add', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(watched ? { artist_id: best.id } : { artist_id: best.id, artist_name: n.name, source: best.source }), + }); + if (!resp.ok) { showToast('Failed to update watchlist', 'error'); return; } + if (watched) { + _artMap._watchSet.delete(best.id); + if (n.type === 'watchlist') n.type = 'similar'; + } else { + _artMap._watchSet.add(best.id); + } + showToast(watched ? `Removed ${n.name} from watchlist` : `Added ${n.name} to watchlist`, watched ? 'info' : 'success'); + _artMapRenderWatchBtn(n); + } catch (e) { + showToast('Failed to update watchlist', 'error'); + } +} + +// Lazily confirm watchlist membership from the server, then refresh the button. +function _artMapCheckWatched(n) { + const best = _artMapNodeBest(n); + _artMap._watchSet = _artMap._watchSet || new Set(); + _artMap._watchChecked = _artMap._watchChecked || new Set(); + if (!best.id || _artMap._watchChecked.has(best.id)) return; + fetch('/api/watchlist/check', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ artist_id: best.id }), + }).then(r => r.json()).then(d => { + _artMap._watchChecked.add(best.id); + if (d && d.success) { + if (d.is_watching) _artMap._watchSet.add(best.id); else _artMap._watchSet.delete(best.id); + if (_artMap._panelArtistId === n.id) _artMapRenderWatchBtn(n); + } + }).catch(() => { }); +} + function _artMapEnsurePanel() { const container = document.getElementById('artist-map-container'); if (!container) return null; @@ -5401,14 +5468,18 @@ function _artMapEnsurePanel() { if (!p) { p = document.createElement('div'); p.id = 'artmap-info-panel'; - p.style.cssText = `position:absolute;top:0;right:0;width:${_artMap._panelW}px;height:100%;` - + `background:linear-gradient(180deg,rgba(20,15,34,0.92),rgba(11,8,20,0.96));backdrop-filter:blur(16px);` - + `border-left:1px solid rgba(168,85,247,0.18);z-index:26;display:flex;flex-direction:column;` - + `color:#fff;overflow:hidden;box-shadow:-10px 0 36px rgba(0,0,0,0.45);font-size:13px;`; p.innerHTML = `
` + `
`; container.appendChild(p); } + // Start below the toolbar so it never covers the navbar (measured each call + // in case the toolbar height changes). + const tb = container.querySelector('.artist-map-toolbar'); + const top = tb ? tb.offsetHeight : 56; + p.style.cssText = `position:absolute;top:${top}px;right:0;width:${_artMap._panelW}px;height:calc(100% - ${top}px);` + + `background:linear-gradient(180deg,rgba(20,15,34,0.92),rgba(11,8,20,0.96));backdrop-filter:blur(16px);` + + `border-left:1px solid rgba(168,85,247,0.18);z-index:20;display:flex;flex-direction:column;` + + `color:#fff;overflow:hidden;box-shadow:-10px 0 36px rgba(0,0,0,0.45);font-size:13px;`; return p; } @@ -5524,7 +5595,7 @@ function _artMapPanelArtist(node) {
- + ${_artMapWatchBtnHtml(node)}
${best.id ? `Open artist page` : ''}
`; @@ -5533,6 +5604,10 @@ function _artMapPanelArtist(node) { const c = document.getElementById('artmap-card-canvas'); if (c) { try { c.getContext('2d').drawImage(bmp, 0, 0, 120, 120); } catch (e) { /* ignore */ } } } + + // Confirm watchlist membership from the server (refreshes the button if it + // differs from the optimistic guess). + _artMapCheckWatched(node); } // Top-list / external entry: show a node's card by id (also ripples it on the map). @@ -7369,7 +7444,18 @@ function _artMapSetupInteraction(canvas) { _artMap.hoveredNode = _artMapHitTest(nx, ny); canvas.style.cursor = _artMap.hoveredNode ? 'pointer' : 'grab'; _artMapShowTooltip(e, _artMap.hoveredNode); - if (_artMap.hoveredNode) _artMapPanelArtist(_artMap.hoveredNode); // rich card in side panel + if (prev !== _artMap.hoveredNode) { + // Debounce the side-panel card: only swap to a bubble you've + // settled on for ~0.8s, so sweeping toward the panel doesn't keep + // changing the card on bubbles you pass over en route. + clearTimeout(_artMap._panelTimer); + if (_artMap.hoveredNode) { + const target = _artMap.hoveredNode; + _artMap._panelTimer = setTimeout(() => { + if (_artMap.hoveredNode === target) _artMapPanelArtist(target); + }, 800); + } + } if (prev !== _artMap.hoveredNode) { // Reset constellation highlight timer clearTimeout(_artMap._constellationTimer); @@ -7400,13 +7486,15 @@ function _artMapSetupInteraction(canvas) { isPanning = false; if (!wasDrag && clickStart) { - // It was a click — emit a water ripple (shoves nearby bubbles) and, - // if it landed on an artist, open it after the ripple kicks off. + // A click is a deliberate select — ripple it and pin its card in the + // side panel immediately (bypassing the hover debounce). The card's + // Details button opens the full modal; click no longer auto-opens it. const { nx, ny } = _artMapScreenToWorld(e, canvas); const node = _artMapHitTest(nx, ny); _artMapEmitRipple(node ? node.x : nx, node ? node.y : ny, node ? node._hue : null); - if (node && (node.spotify_id || node.itunes_id || node.deezer_id)) { - setTimeout(() => openYourArtistInfoModal_direct(node), 200); + if (node) { + clearTimeout(_artMap._panelTimer); + _artMapPanelArtist(node); } } From e00589f40a1363b198e230878c111b9ae3296ae6 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 14:00:07 -0700 Subject: [PATCH 065/108] Artist Map v2: mobile responsive (bottom-sheet panel, full-width map, resize reflow) - Info panel becomes a bottom SHEET on phones (<=760px): slides up when you tap a bubble, doesn't steal map width (islands frame full-width via _artMapReservedW = 0 on mobile). Grip/handle to dismiss; a floating menu FAB opens it to the dashboard + top-artists. Desktop stays the right sidebar. - Genre sidebar hidden on mobile (the top-left quick-jump nav handles genre switching; no room for a sidebar). - Touch tap now selects a bubble (card in the sheet) instead of opening the modal, matching desktop click; ignores taps that were drags. - Resize/orientation: debounced reflow that re-styles the panel for the new breakpoint, recomputes canvas size (minus sidebar/toolbar), and re-frames the focused island / fit. 64 JS integrity tests pass. --- webui/static/discover.js | 137 +++++++++++++++++++++++++++++++-------- 1 file changed, 111 insertions(+), 26 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index b601274f..cd8b9270 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5230,7 +5230,7 @@ function _artMapFitToContent(marginPx = 120) { minY = Math.min(minY, n.y - n.radius); maxY = Math.max(maxY, n.y + n.radius); } if (!isFinite(minX)) return; - const usableW = Math.max(200, _artMap.width - _artMap._panelW); + const usableW = Math.max(200, _artMap.width - _artMapReservedW()); const mapW = maxX - minX + marginPx * 2, mapH = maxY - minY + marginPx * 2; _artMap.zoom = Math.min(usableW / mapW, _artMap.height / mapH, 1); _artMap.offsetX = usableW / 2 - ((minX + maxX) / 2) * _artMap.zoom; @@ -5259,7 +5259,7 @@ function _artMapFocusIsland(idx, opts = {}) { } // Frame the island in the space LEFT of the info panel (~80% of it). - const usableW = Math.max(200, _artMap.width - _artMap._panelW); + const usableW = Math.max(200, _artMap.width - _artMapReservedW()); const span = (isl.r * 2.3) + 120; const z = Math.min(usableW / span, _artMap.height / span, 1.2); _artMap.zoom = z; @@ -5380,7 +5380,18 @@ function _artMapJumpIsland(i) { // ── Right-side info panel ────────────────────────────────────────────────── // A polished detail panel: a discovery dashboard + current-view coverage at the // top, a clickable top-artists list, and a rich artist card when you hover/click -// a bubble. Lives on the right so it never collides with the genre sidebar. +// a bubble. On desktop it's a right sidebar; on mobile it's a bottom sheet that +// slides up on tap and doesn't steal map width. + +function _artMapIsMobile() { + return (window.innerWidth || document.documentElement.clientWidth || 9999) <= 760; +} + +// Horizontal space the panel reserves when framing islands — none on mobile +// (the bottom sheet overlays instead of sitting beside the map). +function _artMapReservedW() { + return _artMapIsMobile() ? 0 : _artMap._panelW; +} function _artMapNodeBest(n) { const map = [['spotify_id', 'spotify'], ['itunes_id', 'itunes'], ['deezer_id', 'deezer'], ['discogs_id', 'discogs'], ['musicbrainz_id', 'musicbrainz']]; @@ -5468,25 +5479,74 @@ function _artMapEnsurePanel() { if (!p) { p = document.createElement('div'); p.id = 'artmap-info-panel'; - p.innerHTML = `
` - + `
`; + p.innerHTML = `
` + + `
` + + `
`; container.appendChild(p); } - // Start below the toolbar so it never covers the navbar (measured each call - // in case the toolbar height changes). - const tb = container.querySelector('.artist-map-toolbar'); - const top = tb ? tb.offsetHeight : 56; - p.style.cssText = `position:absolute;top:${top}px;right:0;width:${_artMap._panelW}px;height:calc(100% - ${top}px);` - + `background:linear-gradient(180deg,rgba(20,15,34,0.92),rgba(11,8,20,0.96));backdrop-filter:blur(16px);` - + `border-left:1px solid rgba(168,85,247,0.18);z-index:20;display:flex;flex-direction:column;` - + `color:#fff;overflow:hidden;box-shadow:-10px 0 36px rgba(0,0,0,0.45);font-size:13px;`; + const base = `background:linear-gradient(180deg,rgba(20,15,34,0.95),rgba(11,8,20,0.97));backdrop-filter:blur(16px);` + + `z-index:22;display:flex;flex-direction:column;color:#fff;overflow:hidden;font-size:13px;`; + const grip = document.getElementById('artmap-panel-grip'); + if (_artMapIsMobile()) { + // Bottom sheet — overlays the map, slides up/down, doesn't steal width. + const open = !!_artMap._panelOpen; + p.style.cssText = base + `position:absolute;left:0;right:0;bottom:0;width:auto;max-height:62%;` + + `border-top:1px solid rgba(168,85,247,0.25);border-radius:18px 18px 0 0;` + + `box-shadow:0 -10px 36px rgba(0,0,0,0.5);transition:transform 0.28s cubic-bezier(.4,0,.2,1);` + + `transform:translateY(${open ? '0' : '100%'});`; + if (grip) grip.style.cssText = `flex:none;height:22px;display:flex;align-items:center;justify-content:center;cursor:grab;` + + `position:relative;` ; + if (grip && !grip.dataset.wired) { + grip.dataset.wired = '1'; + grip.onclick = () => _artMapTogglePanelSheet(false); + grip.innerHTML = ``; + } + } else { + // Right sidebar — below the toolbar so it never covers the navbar. + const tb = container.querySelector('.artist-map-toolbar'); + const top = tb ? tb.offsetHeight : 56; + p.style.cssText = base + `position:absolute;top:${top}px;right:0;width:${_artMap._panelW}px;height:calc(100% - ${top}px);` + + `border-left:1px solid rgba(168,85,247,0.18);box-shadow:-10px 0 36px rgba(0,0,0,0.45);transform:none;`; + if (grip) grip.style.display = 'none'; + } return p; } +// Mobile bottom-sheet open/close. +function _artMapTogglePanelSheet(open) { + _artMap._panelOpen = open; + const p = document.getElementById('artmap-info-panel'); + if (p && _artMapIsMobile()) p.style.transform = `translateY(${open ? '0' : '100%'})`; + _artMapEnsureStatsFab(); +} + +// A floating button (mobile only) to open the panel to the dashboard/top list. +function _artMapEnsureStatsFab() { + const container = document.getElementById('artist-map-container'); + if (!container) return; + let fab = document.getElementById('artmap-stats-fab'); + if (!_artMapIsMobile()) { if (fab) fab.remove(); return; } + if (!fab) { + fab = document.createElement('button'); + fab.id = 'artmap-stats-fab'; + fab.innerHTML = '☰'; + fab.title = 'View stats & artists'; + fab.style.cssText = `position:absolute;right:14px;bottom:14px;width:46px;height:46px;border-radius:50%;` + + `background:linear-gradient(135deg,#7c3aed,#a855f7);border:none;color:#fff;font-size:18px;` + + `box-shadow:0 6px 20px rgba(0,0,0,0.5);z-index:21;cursor:pointer;`; + fab.onclick = () => { _artMap._panelArtistId = null; _artMapRefreshPanel(); _artMapTogglePanelSheet(true); }; + container.appendChild(fab); + } + fab.style.display = _artMap._panelOpen ? 'none' : 'flex'; +} + function _artMapClosePanel() { const p = document.getElementById('artmap-info-panel'); if (p) p.remove(); + const fab = document.getElementById('artmap-stats-fab'); + if (fab) fab.remove(); _artMap._panelArtistId = null; + _artMap._panelOpen = false; } // Stat tile helper @@ -5501,6 +5561,7 @@ function _miniStat(label, value, hue) { function _artMapRefreshPanel() { const p = _artMapEnsurePanel(); if (!p) return; + _artMapEnsureStatsFab(); const head = document.getElementById('artmap-panel-head'); const body = document.getElementById('artmap-panel-body'); if (!head || !body) return; @@ -5608,6 +5669,9 @@ function _artMapPanelArtist(node) { // Confirm watchlist membership from the server (refreshes the button if it // differs from the optimistic guess). _artMapCheckWatched(node); + + // On mobile, sliding the bottom sheet up reveals the card. + if (_artMapIsMobile()) _artMapTogglePanelSheet(true); } // Top-list / external entry: show a node's card by id (also ripples it on the map). @@ -6852,10 +6916,13 @@ async function _openGenreMapWithSelection(selectedGenre) { } container.style.display = 'flex'; - // Show + populate genre sidebar + // Show + populate genre sidebar (hidden on mobile — the top-left quick-jump + // nav handles genre switching there, and there's no room for a sidebar). const sidebar = document.getElementById('artmap-genre-sidebar'); const genreListData = window._artMapGenreList || window._artMapGenreData; - if (sidebar && genreListData?.genres) { + if (sidebar && _artMapIsMobile()) { + sidebar.style.display = 'none'; + } else if (sidebar && genreListData?.genres) { sidebar.style.display = 'flex'; const list = document.getElementById('artmap-genre-sidebar-list'); if (list) { @@ -7556,25 +7623,43 @@ function _artMapSetupInteraction(canvas) { const wx = (t.clientX - rect.left - _artMap.offsetX) / _artMap.zoom; const wy = (t.clientY - rect.top - _artMap.offsetY) / _artMap.zoom; const node = _artMapHitTest(wx, wy); - _artMapEmitRipple(node ? node.x : wx, node ? node.y : wy, node ? node._hue : null); - if (node && (node.spotify_id || node.itunes_id || node.deezer_id)) { - openYourArtistInfoModal_direct(node); + const moved = Math.abs(t.clientX - (lastTouches[0].clientX)) > 8 || Math.abs(t.clientY - (lastTouches[0].clientY)) > 8; + if (!moved) { + _artMapEmitRipple(node ? node.x : wx, node ? node.y : wy, node ? node._hue : null); + if (node) _artMapPanelArtist(node); // tap selects → card in the bottom sheet } } lastTouches = null; }, { passive: false }); - // Handle resize + // Handle resize / orientation change — debounced, and re-frames for the new + // breakpoint (mobile bottom-sheet ⇄ desktop sidebar). window.addEventListener('resize', () => { const container = document.getElementById('artist-map-container'); if (!container || container.style.display === 'none') return; - _artMap.width = container.clientWidth; - _artMap.height = container.clientHeight - 50; - canvas.width = _artMap.width * window.devicePixelRatio; - canvas.height = _artMap.height * window.devicePixelRatio; - canvas.style.width = _artMap.width + 'px'; - canvas.style.height = _artMap.height + 'px'; - _artMap.ctx.scale(window.devicePixelRatio, window.devicePixelRatio); + clearTimeout(_artMap._resizeTimer); + _artMap._resizeTimer = setTimeout(() => { + if (container.style.display === 'none') return; + const sb = document.getElementById('artmap-genre-sidebar'); + const sbW = (sb && sb.style.display !== 'none') ? (sb.offsetWidth || 0) : 0; + const tb = container.querySelector('.artist-map-toolbar'); + _artMap.width = Math.max(120, container.clientWidth - sbW); + _artMap.height = Math.max(120, container.clientHeight - (tb ? tb.offsetHeight : 50)); + canvas.width = _artMap.width * window.devicePixelRatio; // resets ctx transform + canvas.height = _artMap.height * window.devicePixelRatio; + canvas.style.width = _artMap.width + 'px'; + canvas.style.height = _artMap.height + 'px'; + _artMap.ctx.scale(window.devicePixelRatio, window.devicePixelRatio); + _artMapEnsurePanel(); // re-style sidebar ⇄ bottom sheet + if (_artMap._oneIsland && (_artMap._islands || []).length) { + _artMapFocusIsland(_artMap._focusIdx || 0, { bloom: false }); + } else { + _artMapFitToContent(); + _artMapRefreshPanel(); + } + _artMap.dirty = true; + _artMapRender(); + }, 160); }); } From c5e12b904f98a54878010ed72e18083a202736ce Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 14:14:07 -0700 Subject: [PATCH 066/108] Artist Map v2: mobile-responsive toolbar/header - Toolbar wraps on phones (<=760px): back + title + stats and the compact tools stay on row 1, the search drops to its own full-width row below so nothing gets crushed. Brand text hidden, stats truncate with ellipsis. - Island nav + canvas height now MEASURE the toolbar height instead of assuming ~50px, so the taller wrapped header doesn't overlap the nav or clip the canvas. 64 JS integrity tests pass. --- webui/static/discover.js | 13 +++++++++---- webui/static/style.css | 13 +++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/webui/static/discover.js b/webui/static/discover.js index cd8b9270..19ee8809 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -5322,10 +5322,13 @@ function _artMapUpdateIslandNav() { nav.id = 'artmap-island-nav'; container.appendChild(nav); } - // Position top-left, clearing the genre sidebar (when shown) and the toolbar. + // Position top-left, clearing the genre sidebar (when shown) and the toolbar + // (measured — the toolbar wraps taller on mobile). const sb = document.getElementById('artmap-genre-sidebar'); const sbW = (sb && sb.style.display !== 'none') ? (sb.offsetWidth || 0) : 0; - nav.style.cssText = `position:absolute;top:64px;left:${sbW + 16}px;display:flex;align-items:center;gap:12px;padding:7px 12px;background:rgba(16,12,28,0.82);backdrop-filter:blur(10px);border:1px solid rgba(168,85,247,0.25);border-radius:14px;z-index:30;box-shadow:0 6px 24px rgba(0,0,0,0.45);user-select:none;`; + const tb = document.querySelector('.artist-map-toolbar'); + const top = (tb ? tb.offsetHeight : 56) + 10; + nav.style.cssText = `position:absolute;top:${top}px;left:${sbW + 16}px;display:flex;align-items:center;gap:12px;padding:7px 12px;background:rgba(16,12,28,0.82);backdrop-filter:blur(10px);border:1px solid rgba(168,85,247,0.25);border-radius:14px;z-index:30;box-shadow:0 6px 24px rgba(0,0,0,0.45);user-select:none;max-width:calc(100vw - ${sbW + 32}px);`; const idx = _artMap._focusIdx || 0; const isl = islands[idx]; const btn = 'width:30px;height:30px;border-radius:50%;border:1px solid rgba(255,255,255,0.18);background:rgba(255,255,255,0.06);color:#fff;font-size:13px;cursor:pointer;display:flex;align-items:center;justify-content:center;flex:none;'; @@ -5697,7 +5700,8 @@ async function openArtistMap() { _artMap.canvas = canvas; _artMap.ctx = canvas.getContext('2d'); _artMap.width = container.clientWidth; - _artMap.height = container.clientHeight - 50; + const _wtb = container.querySelector('.artist-map-toolbar'); + _artMap.height = container.clientHeight - (_wtb ? _wtb.offsetHeight : 50); canvas.width = _artMap.width * window.devicePixelRatio; canvas.height = _artMap.height * window.devicePixelRatio; canvas.style.width = _artMap.width + 'px'; @@ -7067,7 +7071,8 @@ async function _openArtistMapExplorerWithName(name) { _artMap.canvas = canvas; _artMap.ctx = canvas.getContext('2d'); _artMap.width = container.clientWidth; - _artMap.height = container.clientHeight - 50; + const _wtb = container.querySelector('.artist-map-toolbar'); + _artMap.height = container.clientHeight - (_wtb ? _wtb.offsetHeight : 50); canvas.width = _artMap.width * window.devicePixelRatio; canvas.height = _artMap.height * window.devicePixelRatio; canvas.style.width = _artMap.width + 'px'; diff --git a/webui/static/style.css b/webui/static/style.css index 35afbdef..3abe87c9 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -34248,6 +34248,19 @@ div.artist-hero-badge { .artmap-tool-btn.artmap-zoom { border: none; background: none; width: 30px; padding: 0; justify-content: center; border-radius: 6px; } .artmap-tool-btn.artmap-zoom:hover { background: rgba(255,255,255,0.06); } +/* Mobile: the toolbar wraps — back/title/stats + compact tools on row 1, the + search drops to its own full-width row below so nothing gets crushed. */ +@media (max-width: 760px) { + .artist-map-toolbar { flex-wrap: wrap; padding: 8px 10px; gap: 8px; } + .artmap-nav-left { flex: 1 1 auto; min-width: 0; gap: 6px; } + .artmap-brand-text { display: none; } + .artmap-brand { flex: none; } + .artmap-stats { min-width: 0; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } + .artmap-nav-right { flex: 0 0 auto; gap: 3px; } + .artmap-nav-center { order: 3; flex: 1 1 100%; max-width: none; margin: 0; } + .artmap-search-wrap { width: 100%; } +} + /* Shortcuts modal */ .artmap-shortcuts-modal { background: rgba(20,20,32,0.95); border-radius: 14px; width: 340px; max-width: 90vw; From c8626b1e83feea3caa7a6b2458ce89fc6b8c1e75 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 14:32:44 -0700 Subject: [PATCH 067/108] Artist Map v2: offset toolbar back button clear of the mobile hamburger menu The fixed hamburger (top:16 left:16, 44px, z9999) sat on top of the map's back button on mobile. Push .artmap-nav-left right by 52px on <=760px so the back button clears it. --- webui/static/style.css | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webui/static/style.css b/webui/static/style.css index 3abe87c9..d50d70e9 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -34252,7 +34252,9 @@ div.artist-hero-badge { search drops to its own full-width row below so nothing gets crushed. */ @media (max-width: 760px) { .artist-map-toolbar { flex-wrap: wrap; padding: 8px 10px; gap: 8px; } - .artmap-nav-left { flex: 1 1 auto; min-width: 0; gap: 6px; } + /* Clear the fixed hamburger menu (top:16 left:16, 44px wide) so the map's + back button isn't hidden under it. */ + .artmap-nav-left { flex: 1 1 auto; min-width: 0; gap: 6px; margin-left: 52px; } .artmap-brand-text { display: none; } .artmap-brand { flex: none; } .artmap-stats { min-width: 0; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } From 89e3486e8460da59246398215fe6eb14e0de6f8f Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 15:07:49 -0700 Subject: [PATCH 068/108] =?UTF-8?q?Similar=20Artists=20enrichment=20worker?= =?UTF-8?q?=20(MusicMap=20=E2=86=92=20match=20=E2=86=92=20store)=20for=20l?= =?UTF-8?q?ibrary=20artists?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the gap where similar artists only existed for WATCHLIST artists: a new background worker populates them for the whole LIBRARY, slotting into the existing enrichment-worker pattern (bubble + Manage Enrichment Workers modal, status/pause/resume, matched/not_found/pending/errors). Per source-matched library artist → get_musicmap_similar_artists(name, 25) (the same matcher the artist-detail page uses: fetches MusicMap names, matches each to the user's source chain — primary + active fallbacks — returns only matched artists) → store via add_or_update_similar_artist keyed by the artist's metadata source id, the SAME key the watchlist scanner + artist map use, so the two cooperate (idempotent upsert + retry_days window). - core/similar_artists_worker.py: pure seams (pick_source_artist_id, map_payload_to_store_kwargs, process_artist) + the threaded worker; skips artists not yet source-matched; classifies not_found vs transient error (retry after 30d). - DB migration: similar_artists_match_status / _last_attempted on artists (mirrors every other source worker's tracking columns). - Registered in EnrichmentService + instantiated in web_server, DEFAULT-PAUSED (opt-in) like Amazon — MusicMap is scraped/outage-prone + this is library-wide. - SERVICE_ENTITY_SUPPORT['similar_artists']=('artist',) so the modal breakdown ('artists with / without similars') + Retry work; manual-match (inapplicable to a relationship) is gated out via relationship:true. - 10 seam tests; existing 80 enrichment tests still pass. Note: keys under profile 1 (single-profile setups); multi-profile is future work. --- core/enrichment/unmatched.py | 6 + core/similar_artists_worker.py | 296 +++++++++++++++++++++++++++ database/music_database.py | 25 +++ tests/test_similar_artists_worker.py | 127 ++++++++++++ web_server.py | 26 +++ webui/static/enrichment-manager.js | 13 +- 6 files changed, 491 insertions(+), 2 deletions(-) create mode 100644 core/similar_artists_worker.py create mode 100644 tests/test_similar_artists_worker.py diff --git a/core/enrichment/unmatched.py b/core/enrichment/unmatched.py index 1bcd5eae..0d63ac79 100644 --- a/core/enrichment/unmatched.py +++ b/core/enrichment/unmatched.py @@ -32,6 +32,12 @@ SERVICE_ENTITY_SUPPORT = { 'tidal': ('artist', 'album', 'track'), 'qobuz': ('artist', 'album', 'track'), 'amazon': ('artist', 'album', 'track'), + # Relationship enrichment (not a metadata source): the Similar Artists worker + # only operates at the artist level, and its _match_status tracks + # whether MusicMap similars were fetched (not a source-id match). So the + # breakdown / unmatched list here means "artists we have / don't have + # similars for" — informative, even though there's no manual-match action. + 'similar_artists': ('artist',), } # entity_type -> table / display-name column / image expression / optional join diff --git a/core/similar_artists_worker.py b/core/similar_artists_worker.py new file mode 100644 index 00000000..93db6416 --- /dev/null +++ b/core/similar_artists_worker.py @@ -0,0 +1,296 @@ +"""Background worker that fills the ``similar_artists`` table for LIBRARY artists. + +The watchlist scanner only populates similar artists for *watchlist* artists, so +the artist map / discover surfaces are rich for watchlisted artists and sparse +for the rest of the library. This worker closes that gap: for every +source-matched library artist it asks MusicMap for ~25 similar artists, matches +each one to the user's metadata source chain (primary + active fallbacks) via the +shared :func:`core.metadata.similar_artists.get_musicmap_similar_artists`, and +stores the matched results keyed by the library artist's **metadata source id** — +the same key the watchlist scanner and the artist map use, so the two cooperate +(idempotent upsert + a retry window keep them from double-fetching). + +It plugs into the existing enrichment-worker pattern (background thread, status / +pause / resume, ``matched / not_found / pending / errors`` stats) so it shows up +as a bubble in the dashboard / Manage Enrichment Workers modal like every other +source worker. + +The pure seams below (:func:`pick_source_artist_id`, +:func:`map_payload_to_store_kwargs`, :func:`process_artist`) carry the logic and +are unit-tested in isolation; the class wires them to the DB + MusicMap. +""" + +from __future__ import annotations + +import threading +from typing import Any, Callable, Dict, Optional + +from utils.logging_config import get_logger +from core.worker_utils import interruptible_sleep + +logger = get_logger("similar_artists_worker") + + +# A matched MusicMap payload is {id, source, name, image_url, genres, popularity}. +# Map its single (id, source) onto the right add_or_update_similar_artist id +# kwarg. The table only has columns for these four providers; a match on any +# other source (e.g. discogs) is still stored, name-only. +_SOURCE_ID_FIELD = { + 'spotify': 'similar_artist_spotify_id', + 'itunes': 'similar_artist_itunes_id', + 'deezer': 'similar_artist_deezer_id', + 'musicbrainz': 'similar_artist_musicbrainz_id', +} + +# Library-artist source-id columns, in the same priority the watchlist scanner +# uses to key its rows — so a library artist and (if also watchlisted) its +# watchlist row resolve to the SAME source_artist_id and don't duplicate work. +_LIBRARY_ID_COLUMNS = ('spotify_artist_id', 'itunes_artist_id', 'deezer_id', 'musicbrainz_id') + + +def map_payload_to_store_kwargs(payload: Dict[str, Any]) -> Dict[str, Optional[str]]: + """Turn a matched MusicMap payload into the id kwarg for the store call.""" + src = str(payload.get('source') or '').lower() + pid = str(payload.get('id') or '') + field = _SOURCE_ID_FIELD.get(src) + return {field: pid} if (field and pid) else {} + + +def pick_source_artist_id(row: Dict[str, Any]) -> Optional[str]: + """The metadata source id to key a library artist's similars by, or None if + the artist isn't matched to any metadata source yet (→ skip it).""" + for key in _LIBRARY_ID_COLUMNS: + v = row.get(key) + if v: + return str(v) + return None + + +def process_artist( + source_artist_id: str, + artist_name: str, + fetch_similars: Callable[[str, int], Dict[str, Any]], + store_similar: Callable[..., bool], + limit: int = 25, + profile_id: int = 1, +) -> tuple: + """Fetch + store similar artists for one library artist. + + ``fetch_similars(name, limit)`` returns the ``get_musicmap_similar_artists`` + payload; ``store_similar(**kwargs)`` is ``add_or_update_similar_artist``. + Returns ``(status, stored_count)`` where status is one of: + - ``'matched'`` — stored ≥1 similar artist + - ``'not_found'`` — MusicMap had no entry / nothing matched a source + - ``'error'`` — MusicMap/source failure (transient; eligible for retry) + """ + try: + result = fetch_similars(artist_name, limit) or {} + except Exception as exc: + logger.debug("MusicMap fetch raised for %s: %s", artist_name, exc) + return ('error', 0) + + if not result.get('success'): + # 404/400 = genuinely no MusicMap entry → 'not_found' (don't keep retrying); + # anything else (timeout, 5xx, no providers) = transient → 'error' (retry). + code = result.get('status_code') + return ('not_found' if code in (400, 404) else 'error', 0) + + sims = result.get('similar_artists') or [] + if not sims: + return ('not_found', 0) + + stored = 0 + for rank, s in enumerate(sims, 1): + name = s.get('name') + if not name: + continue + kwargs = map_payload_to_store_kwargs(s) + try: + ok = store_similar( + source_artist_id=source_artist_id, + similar_artist_name=name, + similarity_rank=rank, + profile_id=profile_id, + image_url=s.get('image_url'), + genres=s.get('genres'), + popularity=s.get('popularity', 0) or 0, + **kwargs, + ) + if ok: + stored += 1 + except Exception as exc: + logger.debug("store similar failed for %s: %s", name, exc) + + return ('matched' if stored else 'not_found', stored) + + +class SimilarArtistsWorker: + """Background worker that populates similar artists for library artists.""" + + def __init__(self, database): + self.db = database + self.running = False + self.paused = False + self.should_stop = False + self.thread: Optional[threading.Thread] = None + self._stop_event = threading.Event() + self.current_item: Optional[str] = None + self.stats = {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0} + self.retry_days = 30 + self.limit = 25 + # similar_artists rows are profile-scoped; the library is shared. v1 keys + # under the default profile (matches single-profile setups, which is the + # common case). Multi-profile per-source-chain population is future work. + self.profile_id = 1 + logger.info("Similar Artists background worker initialized") + + # ── lifecycle (mirrors the other enrichment workers) ────────────────── + def start(self): + if self.running: + return + self.running = True + self.should_stop = False + self._stop_event.clear() + self.thread = threading.Thread(target=self._run, daemon=True) + self.thread.start() + logger.info("Similar Artists worker started") + + def stop(self): + self.should_stop = True + self.running = False + self._stop_event.set() + + def pause(self): + self.paused = True + + def resume(self): + self.paused = False + + def get_stats(self) -> Dict[str, Any]: + try: + self.stats['pending'] = self._count_pending() + except Exception: + pass + is_running = self.running and (self.thread is not None and self.thread.is_alive()) + is_idle = is_running and not self.paused and self.stats['pending'] == 0 and self.current_item is None + return { + 'enabled': True, + 'running': is_running and not self.paused, + 'paused': self.paused, + 'idle': is_idle, + 'current_item': self.current_item, + 'stats': self.stats.copy(), + 'progress': {}, + } + + # ── worker loop ──────────────────────────────────────────────────────── + def _run(self): + logger.info("Similar Artists worker thread started") + # Imported lazily so the worker module stays import-light for tests. + from core.metadata.similar_artists import get_musicmap_similar_artists + + while not self.should_stop: + try: + if self.paused: + interruptible_sleep(self._stop_event, 1) + continue + + self.current_item = None + artist = self._get_next_artist() + if not artist: + interruptible_sleep(self._stop_event, 15) + continue + + sid = pick_source_artist_id(artist) + if not sid: + # Query already filters to artists with a source id; guard anyway. + self._mark(artist['id'], 'error') + continue + + self.current_item = artist.get('name') + status, count = process_artist( + sid, artist['name'], + get_musicmap_similar_artists, + self.db.add_or_update_similar_artist, + limit=self.limit, profile_id=self.profile_id, + ) + self._mark(artist['id'], status) + if status == 'matched': + self.stats['matched'] += 1 + logger.debug("Similar artists: %s → stored %d", artist['name'], count) + elif status == 'not_found': + self.stats['not_found'] += 1 + else: + self.stats['errors'] += 1 + + # Pace MusicMap (name search per candidate is heavy + rate-limited). + interruptible_sleep(self._stop_event, 3) + except Exception as exc: + logger.error("Similar Artists worker loop error: %s", exc) + interruptible_sleep(self._stop_event, 5) + + logger.info("Similar Artists worker thread finished") + + # ── DB helpers (thin; the testable logic lives in the pure seams above) ── + def _has_source_id_clause(self) -> str: + return '(' + ' OR '.join(f"{c} IS NOT NULL AND {c} != ''" for c in _LIBRARY_ID_COLUMNS) + ')' + + def _get_next_artist(self) -> Optional[Dict[str, Any]]: + try: + conn = self.db._get_connection() + cursor = conn.cursor() + cols = "id, name, " + ", ".join(_LIBRARY_ID_COLUMNS) + have_id = self._has_source_id_clause() + # 1) Unattempted source-matched artists. + cursor.execute(f""" + SELECT {cols} FROM artists + WHERE id IS NOT NULL AND name IS NOT NULL + AND similar_artists_match_status IS NULL + AND {have_id} + ORDER BY id ASC LIMIT 1 + """) + row = cursor.fetchone() + # 2) Retry transient failures (and re-check 'not_found') after retry_days. + if not row: + cursor.execute(f""" + SELECT {cols} FROM artists + WHERE id IS NOT NULL AND name IS NOT NULL + AND similar_artists_match_status IN ('error', 'not_found') + AND (similar_artists_last_attempted IS NULL + OR similar_artists_last_attempted < datetime('now', ?)) + AND {have_id} + ORDER BY similar_artists_last_attempted ASC LIMIT 1 + """, (f'-{self.retry_days} days',)) + row = cursor.fetchone() + if not row: + return None + keys = ['id', 'name'] + list(_LIBRARY_ID_COLUMNS) + return dict(zip(keys, row)) + except Exception as exc: + logger.debug("Similar Artists _get_next_artist failed: %s", exc) + return None + + def _mark(self, artist_id, status: str): + try: + conn = self.db._get_connection() + cursor = conn.cursor() + cursor.execute( + "UPDATE artists SET similar_artists_match_status = ?, " + "similar_artists_last_attempted = CURRENT_TIMESTAMP WHERE id = ?", + (status, artist_id), + ) + conn.commit() + except Exception as exc: + logger.debug("Similar Artists _mark failed for %s: %s", artist_id, exc) + + def _count_pending(self) -> int: + try: + conn = self.db._get_connection() + cursor = conn.cursor() + cursor.execute( + f"SELECT COUNT(*) FROM artists WHERE similar_artists_match_status IS NULL " + f"AND {self._has_source_id_clause()}" + ) + return int(cursor.fetchone()[0] or 0) + except Exception: + return 0 diff --git a/database/music_database.py b/database/music_database.py index aceb49c4..3e7cdd67 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -427,6 +427,9 @@ class MusicDatabase: # Add Amazon artist ID column (migration) self._add_amazon_columns(cursor) + # Add Similar-Artists worker tracking columns (migration) + self._add_similar_artists_worker_columns(cursor) + # Backfill match_status for rows that already have an external ID but # NULL status. Prevents enrichment workers from re-processing these # rows forever. Must run AFTER all *_match_status columns have been @@ -2417,6 +2420,28 @@ class MusicDatabase: except Exception as e: logger.error(f"Error adding Discogs columns: {e}") + def _add_similar_artists_worker_columns(self, cursor): + """Add Similar-Artists worker tracking columns to the artists table. + + Mirrors the per-source enrichment pattern: a match_status (NULL = + unattempted, then 'matched'/'not_found'/'error') + last_attempted + timestamp so the SimilarArtistsWorker can pick the next library artist to + fetch MusicMap similars for and retry transient failures after a window. + Idempotent — only adds columns that aren't already present. + """ + try: + cursor.execute("PRAGMA table_info(artists)") + artists_columns = [column[1] for column in cursor.fetchall()] + + if 'similar_artists_match_status' not in artists_columns: + cursor.execute("ALTER TABLE artists ADD COLUMN similar_artists_match_status TEXT") + if 'similar_artists_last_attempted' not in artists_columns: + cursor.execute("ALTER TABLE artists ADD COLUMN similar_artists_last_attempted TIMESTAMP") + + cursor.execute("CREATE INDEX IF NOT EXISTS idx_artists_similarartists_status ON artists (similar_artists_match_status)") + except Exception as e: + logger.error(f"Error adding similar-artists worker columns: {e}") + def _add_amazon_columns(self, cursor): """Add Amazon enrichment tracking columns to artists, albums, and tracks.""" try: diff --git a/tests/test_similar_artists_worker.py b/tests/test_similar_artists_worker.py new file mode 100644 index 00000000..ad23f15d --- /dev/null +++ b/tests/test_similar_artists_worker.py @@ -0,0 +1,127 @@ +"""Seam tests for the Similar-Artists enrichment worker's pure logic. + +The worker fills the similar_artists table for LIBRARY artists (the watchlist +scanner only does watchlist artists). These tests exercise the import-light +seams in isolation — no DB, no MusicMap — via injected fakes: + + - pick_source_artist_id → keying priority (and skip un-matched artists) + - map_payload_to_store_kwargs → MusicMap {id,source} → the right id column + - process_artist → fetch→match→store orchestration + status codes +""" + +from __future__ import annotations + +import core.similar_artists_worker as w + + +# -------------------------------------------------------------------------- +# pick_source_artist_id — which id keys the artist's similars (must match the +# watchlist scanner's priority so both write the SAME source_artist_id). +# -------------------------------------------------------------------------- + +def test_pick_source_artist_id_priority(): + row = {'spotify_artist_id': 'sp1', 'itunes_artist_id': 'it1', 'deezer_id': 'dz1'} + assert w.pick_source_artist_id(row) == 'sp1' # spotify wins + assert w.pick_source_artist_id({'itunes_artist_id': 'it1', 'deezer_id': 'dz1'}) == 'it1' + assert w.pick_source_artist_id({'deezer_id': 'dz1'}) == 'dz1' + assert w.pick_source_artist_id({'musicbrainz_id': 'mb1'}) == 'mb1' + + +def test_pick_source_artist_id_none_when_unmatched(): + # Library artist not matched to any metadata source yet → skip (None). + assert w.pick_source_artist_id({'spotify_artist_id': None, 'itunes_artist_id': ''}) is None + assert w.pick_source_artist_id({}) is None + + +# -------------------------------------------------------------------------- +# map_payload_to_store_kwargs — MusicMap payload {id, source} → store kwarg. +# -------------------------------------------------------------------------- + +def test_map_payload_each_source(): + assert w.map_payload_to_store_kwargs({'id': 'x', 'source': 'spotify'}) == {'similar_artist_spotify_id': 'x'} + assert w.map_payload_to_store_kwargs({'id': 'x', 'source': 'itunes'}) == {'similar_artist_itunes_id': 'x'} + assert w.map_payload_to_store_kwargs({'id': 'x', 'source': 'deezer'}) == {'similar_artist_deezer_id': 'x'} + assert w.map_payload_to_store_kwargs({'id': 'x', 'source': 'musicbrainz'}) == {'similar_artist_musicbrainz_id': 'x'} + + +def test_map_payload_unknown_source_or_no_id(): + # discogs has no column → name-only (empty kwargs), not a crash. + assert w.map_payload_to_store_kwargs({'id': 'x', 'source': 'discogs'}) == {} + assert w.map_payload_to_store_kwargs({'source': 'spotify'}) == {} # no id + + +# -------------------------------------------------------------------------- +# process_artist — fetch → store, status classification, keying. +# -------------------------------------------------------------------------- + +def _capture_store(): + calls = [] + + def store(**kwargs): + calls.append(kwargs) + return True + return store, calls + + +def test_process_artist_matched_stores_with_keying(): + store, calls = _capture_store() + payload = { + 'success': True, + 'similar_artists': [ + {'name': 'B', 'id': 'sp_b', 'source': 'spotify', 'genres': ['rap'], 'popularity': 70}, + {'name': 'C', 'id': 'it_c', 'source': 'itunes', 'image_url': 'http://x'}, + ], + } + status, count = w.process_artist('SRC1', 'A', lambda n, l: payload, store, limit=25, profile_id=1) + assert status == 'matched' and count == 2 + # All similars keyed by the SOURCE artist id we passed (not the library PK). + assert all(c['source_artist_id'] == 'SRC1' for c in calls) + assert all(c['profile_id'] == 1 for c in calls) + # Provider id mapped to the right column; rank preserved in order. + assert calls[0]['similar_artist_spotify_id'] == 'sp_b' and calls[0]['similarity_rank'] == 1 + assert calls[1]['similar_artist_itunes_id'] == 'it_c' and calls[1]['similarity_rank'] == 2 + assert calls[0]['genres'] == ['rap'] and calls[0]['popularity'] == 70 + + +def test_process_artist_not_found_when_no_matches(): + store, calls = _capture_store() + status, count = w.process_artist('S', 'A', lambda n, l: {'success': True, 'similar_artists': []}, store) + assert status == 'not_found' and count == 0 and calls == [] + + +def test_process_artist_not_found_on_404(): + # Genuinely no MusicMap entry — shouldn't be retried as an error. + store, _ = _capture_store() + status, _ = w.process_artist('S', 'A', lambda n, l: {'success': False, 'status_code': 404}, store) + assert status == 'not_found' + + +def test_process_artist_error_on_outage_is_retriable(): + # 5xx / no providers → transient error (worker retries after retry_days). + store, _ = _capture_store() + status, _ = w.process_artist('S', 'A', lambda n, l: {'success': False, 'status_code': 502}, store) + assert status == 'error' + + +def test_process_artist_error_when_fetch_raises(): + store, _ = _capture_store() + + def boom(n, l): + raise RuntimeError('musicmap down') + status, count = w.process_artist('S', 'A', boom, store) + assert status == 'error' and count == 0 + + +def test_process_artist_skips_unstorable_but_counts_real(): + # A store() that fails for one row shouldn't abort the rest. + calls = [] + + def store(**kwargs): + calls.append(kwargs) + return kwargs['similar_artist_name'] != 'B' # B fails to store + payload = {'success': True, 'similar_artists': [ + {'name': 'B', 'id': '1', 'source': 'spotify'}, + {'name': 'C', 'id': '2', 'source': 'spotify'}, + ]} + status, count = w.process_artist('S', 'A', lambda n, l: payload, store) + assert status == 'matched' and count == 1 and len(calls) == 2 diff --git a/web_server.py b/web_server.py index 13eb3368..7136fbb9 100644 --- a/web_server.py +++ b/web_server.py @@ -32889,6 +32889,27 @@ except Exception as e: amazon_worker = None +# --- Similar Artists Worker Initialization --- +# Fills the similar_artists table for LIBRARY artists (the watchlist scanner only +# covers watchlist artists). Opt-in by default like Amazon: it leans on MusicMap +# (a scraped, outage-prone source) and does per-candidate metadata-source +# searches across the whole library, so it stays paused until the user enables it. +similar_artists_worker = None +try: + from core.similar_artists_worker import SimilarArtistsWorker + similar_artists_db = MusicDatabase() + similar_artists_worker = SimilarArtistsWorker(database=similar_artists_db) + similar_artists_worker.start() + if config_manager.get('similar_artists_enrichment_paused', True): + similar_artists_worker.pause() + logger.info("Similar Artists worker initialized (paused — enable it to populate library similars)") + else: + logger.info("Similar Artists worker initialized and started") +except Exception as e: + logger.error(f"Similar Artists worker initialization failed: {e}") + similar_artists_worker = None + + # ================================================================================================ # SPOTIFY ENRICHMENT INTEGRATION # ================================================================================================ @@ -34653,6 +34674,11 @@ _register_enrichment_services([ worker_getter=lambda: amazon_worker, config_paused_key='amazon_enrichment_paused', ), + _EnrichmentService( + id='similar_artists', display_name='Similar Artists', + worker_getter=lambda: similar_artists_worker, + config_paused_key='similar_artists_enrichment_paused', + ), ]) _configure_enrichment_api( diff --git a/webui/static/enrichment-manager.js b/webui/static/enrichment-manager.js index 9e0eb173..94b35299 100644 --- a/webui/static/enrichment-manager.js +++ b/webui/static/enrichment-manager.js @@ -32,6 +32,12 @@ const ENRICHMENT_WORKERS = [ { id: 'tidal', name: 'Tidal', color: '#00cfe6', logoSel: '.tidal-enrich-logo', imgFilter: 'invert(1) brightness(1.8)', imgRound: true }, { id: 'qobuz', name: 'Qobuz', color: '#0070ef', logoSel: '.qobuz-enrich-logo', imgFilter: 'invert(1)', imgRound: true }, { id: 'amazon', name: 'Amazon Music', color: '#ff9900', logoSel: '.amazon-enrich-logo', imgFilter: 'brightness(0) invert(1)' }, + // Relationship worker (MusicMap → similar artists), not a metadata source. + // No dashboard logo element, so it uses a direct MusicMap logo URL. + // relationship:true tells the detail panel to drop the (inapplicable) + // manual-match affordance. + { id: 'similar_artists', name: 'Similar Artists', color: '#a855f7', relationship: true, + logoUrl: 'https://www.music-map.com/elements/objects/og_logo.png', imgRound: true }, ]; const _emWorkerById = Object.fromEntries(ENRICHMENT_WORKERS.map(w => [w.id, w])); @@ -48,7 +54,9 @@ function _emHexToRgb(hex) { // Resolve a worker's logo URL from the live dashboard bubble (null if absent). function _emLogoSrc(workerId) { const w = _emWorkerById[workerId]; - if (!w || !w.logoSel) return null; + if (!w) return null; + if (w.logoUrl) return w.logoUrl; // direct URL (workers w/o a dashboard logo) + if (!w.logoSel) return null; const img = document.querySelector(w.logoSel); return img && img.src ? img.src : null; } @@ -825,7 +833,8 @@ function _emRenderUnmatchedList() {
${statusBadge} ${last}
- + ${(_emWorkerById[id] && _emWorkerById[id].relationship) ? '' + : ``}
From d70d410f6f7bb2ebefb352c36a3ddee769c86287 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 15:19:46 -0700 Subject: [PATCH 069/108] Similar Artists worker: dashboard enrichment bubble Adds the dashboard status bubble (the small icon row) for the Similar Artists worker, alongside the modal entry. Mirrors the per-source bubbles: MusicMap logo, purple accent, spinner + active/complete/paused states, hover tooltip, and a 2s status poll against /api/enrichment/similar_artists/status. Click toggles pause/resume. Tooltip shows matched/pending (the worker has no artist/album/track phases). 74 JS integrity tests pass. --- webui/index.html | 23 ++++++++++++ webui/static/enrichment.js | 74 ++++++++++++++++++++++++++++++++++++++ webui/static/style.css | 68 +++++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+) diff --git a/webui/index.html b/webui/index.html index eb3f5ac6..2d8d7d97 100644 --- a/webui/index.html +++ b/webui/index.html @@ -565,6 +565,29 @@
+ +
+ +
+
+
Similar Artists Enrichment
+
+
Status: Idle +
+
No active matches +
+
Progress: 0 / 0 +
+
+
+
+
@@ -9569,6 +9584,48 @@ function openYouTubeDiscoveryModal(urlHash) { } console.log('✨ Created new modal with current state'); + if (isSpotifyPublic && state.spotify_public_playlist_id && typeof loadPlaylistOrganizePreferenceIntoModal === 'function') { + void loadPlaylistOrganizePreferenceIntoModal( + `spotify_public_${state.spotify_public_playlist_id}`, + 'spotify_public', + ); + } + } +} + +function discoveryModalOrganizeFooterHtml(state) { + if (!state || typeof playlistOrganizeToggleHtml !== 'function') { + return ''; + } + if (state.is_spotify_public_playlist && state.spotify_public_playlist_id) { + const ref = `spotify_public_${state.spotify_public_playlist_id}`; + return `
${playlistOrganizeToggleHtml(ref, 'spotify_public')}
`; + } + return ''; +} + +function buildDiscoveryModalFooterLeftHtml(urlHash, phase, state) { + const organize = discoveryModalOrganizeFooterHtml(state); + const actions = getModalActionButtons(urlHash, phase, state); + return `${organize}
${actions}
`; +} + +function setDiscoveryModalFooterActions(urlHash, phase, state = null) { + if (!state) { + state = listenbrainzPlaylistStates[urlHash] || youtubePlaylistStates[urlHash]; + } + const modal = document.getElementById(`youtube-discovery-modal-${urlHash}`); + const footerLeft = modal?.querySelector('.modal-footer-left'); + if (!footerLeft) { + return; + } + footerLeft.innerHTML = buildDiscoveryModalFooterLeftHtml(urlHash, phase, state); + if (state?.is_spotify_public_playlist && state.spotify_public_playlist_id + && typeof loadPlaylistOrganizePreferenceIntoModal === 'function') { + void loadPlaylistOrganizePreferenceIntoModal( + `spotify_public_${state.spotify_public_playlist_id}`, + 'spotify_public', + ); } } @@ -9650,7 +9707,12 @@ function getModalActionButtons(urlHash, phase, state = null) { } else if (isDeezer) { buttons += ``; } else if (isSpotifyPublic) { - buttons += ``; + const folderArg = _isSoulsyncStandalone ? ', true' : ''; + const label = _isSoulsyncStandalone + ? '📁 Download to Playlist Folder' + : '🔍 Download Missing Tracks'; + const extraClass = _isSoulsyncStandalone ? ' soulsync-standalone-action' : ''; + buttons += ``; } else if (isITunesLink) { buttons += ``; } else if (isBeatport) { @@ -9819,7 +9881,12 @@ function getModalActionButtons(urlHash, phase, state = null) { } else if (isQobuz) { syncCompleteButtons += ``; } else if (isSpotifyPublic) { - syncCompleteButtons += ``; + const folderArg = _isSoulsyncStandalone ? ', true' : ''; + const label = _isSoulsyncStandalone + ? '📁 Download to Playlist Folder' + : '🔍 Download Missing Tracks'; + const extraClass = _isSoulsyncStandalone ? ' soulsync-standalone-action' : ''; + syncCompleteButtons += ``; } else if (isITunesLink) { syncCompleteButtons += ``; } else if (isBeatport) { @@ -9875,7 +9942,12 @@ function getModalActionButtons(urlHash, phase, state = null) { } else if (isDeezer) { dlCompleteButtons += ``; } else if (isSpotifyPublic) { - dlCompleteButtons += ``; + const folderArg = _isSoulsyncStandalone ? ', true' : ''; + const label = _isSoulsyncStandalone + ? '📁 Download to Playlist Folder' + : '🔍 Download Missing Tracks'; + const extraClass = _isSoulsyncStandalone ? ' soulsync-standalone-action' : ''; + dlCompleteButtons += ``; } else if (isITunesLink) { dlCompleteButtons += ``; } else if (isBeatport) { @@ -10118,18 +10190,17 @@ function updateYouTubeDiscoveryModal(urlHash, status) { const state = listenbrainzPlaylistStates[urlHash] || youtubePlaylistStates[urlHash]; if (state && state.phase === 'discovering') { state.phase = 'discovered'; - const actionButtonsContainer = document.querySelector(`#youtube-discovery-modal-${urlHash} .modal-footer-left`); - if (actionButtonsContainer) { - actionButtonsContainer.innerHTML = getModalActionButtons(urlHash, 'discovered', state); - console.log(`✨ Updated action buttons for completed discovery: ${urlHash}`); - } + setDiscoveryModalFooterActions(urlHash, 'discovered', state); + console.log(`✨ Updated action buttons for completed discovery: ${urlHash}`); const descEl = document.querySelector(`#youtube-discovery-modal-${urlHash} .modal-description`); if (descEl) descEl.textContent = 'Discovery complete! View the results below.'; } else if (state && state.phase === 'discovered') { // Already discovered — ensure buttons are correct (e.g. after rehydration) - const actionButtonsContainer = document.querySelector(`#youtube-discovery-modal-${urlHash} .modal-footer-left`); + const actionButtonsContainer = document.querySelector( + `#youtube-discovery-modal-${urlHash} .modal-footer-actions`, + ); if (actionButtonsContainer && actionButtonsContainer.querySelector('.modal-info')) { - actionButtonsContainer.innerHTML = getModalActionButtons(urlHash, 'discovered', state); + setDiscoveryModalFooterActions(urlHash, 'discovered', state); } } } @@ -10626,7 +10697,7 @@ function updateYouTubeModalButtons(urlHash, phase) { const footerLeft = modal.querySelector('.modal-footer-left'); if (footerLeft) { - footerLeft.innerHTML = getModalActionButtons(urlHash, phase); + setDiscoveryModalFooterActions(urlHash, phase); } } diff --git a/webui/static/sync-spotify.js b/webui/static/sync-spotify.js index 77da6514..6af72167 100644 --- a/webui/static/sync-spotify.js +++ b/webui/static/sync-spotify.js @@ -2210,6 +2210,9 @@ async function openDownloadMissingModal(playlistId) { } process.modalElement.style.display = 'flex'; } + if (typeof refreshOrganizePreferenceForDownloadModal === 'function') { + await refreshOrganizePreferenceForDownloadModal(playlistId); + } hideLoadingOverlay(); return; } @@ -2375,7 +2378,9 @@ async function openDownloadMissingModal(playlistId) { Force Download All - + ${typeof downloadMissingModalOrganizeCheckboxHtml === 'function' + ? downloadMissingModalOrganizeCheckboxHtml(playlistId) + : ``} diff --git a/webui/static/style.css b/webui/static/style.css index cea5ca67..42325850 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -64905,6 +64905,7 @@ body.reduce-effects .dash-card::after { background: linear-gradient(135deg, rgba(var(--accent-rgb), 0.9), rgba(var(--accent-rgb), 0.55)); box-shadow: 0 0 12px rgba(var(--accent-rgb), 0.5); } +.em-manage-btn-logo { width: 22px; height: 22px; object-fit: contain; display: block; } .em-manage-btn-label { background: linear-gradient(90deg, #fff, rgba(255,255,255,0.85)); -webkit-background-clip: text; background-clip: text; } .enrichment-manager-modal { diff --git a/webui/static/worker-orbs.js b/webui/static/worker-orbs.js index ad91a09d..1901a129 100644 --- a/webui/static/worker-orbs.js +++ b/webui/static/worker-orbs.js @@ -591,12 +591,16 @@ ctx.fill(); if (hubImageReady) { - // SoulSync logo as the nucleus — sized to the pulsing radius, - // brightness lifts a touch with energy - const imgSize = hubR * 3.2; + // SoulSync logo as the nucleus — fit to the pulsing radius while + // preserving the image's natural aspect ratio (no stretch) + const natW = hubImage.naturalWidth || 1; + const natH = hubImage.naturalHeight || 1; + const fit = (hubR * 3.2) / Math.max(natW, natH); + const dw = natW * fit; + const dh = natH * fit; ctx.save(); ctx.globalAlpha = Math.min(1, 0.85 + energy * 0.15 + slow * 0.1); - ctx.drawImage(hubImage, orb.x - imgSize / 2, orb.y - imgSize / 2, imgSize, imgSize); + ctx.drawImage(hubImage, orb.x - dw / 2, orb.y - dh / 2, dw, dh); ctx.restore(); } else { // Fallback while the logo loads: solid bright core + highlight From 634b9ca0a066bd671eec71404926401119c0af53 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 3 Jun 2026 17:30:54 -0700 Subject: [PATCH 088/108] Worker orbs: cache glow sprites for perf + SoulSync logo in enrichment modal topbar Performance: - Bake one soft glow sprite per colour into an offscreen canvas and blit it with drawImage instead of allocating a radial gradient every frame. This was the hot path: sparks + inbound pulses + every orb glow each built a gradient per frame (100+/frame at 60fps). Colours quantised to 8-step buckets to bound the cache (imperceptible tint shift, keeps the rainbow path from minting a sprite every frame). - Cache each orb's button element at init so the 30-frame active-state check no longer re-runs querySelector. - Net: the pulses/glows look identical, far fewer allocations per frame. UI: - Enrichment manager modal topbar icon now uses the SoulSync logo (trans2.png) instead of the helix emoji, matching the dashboard button. --- webui/static/enrichment-manager.js | 2 +- webui/static/style.css | 1 + webui/static/worker-orbs.js | 80 ++++++++++++++++++------------ 3 files changed, 49 insertions(+), 34 deletions(-) diff --git a/webui/static/enrichment-manager.js b/webui/static/enrichment-manager.js index b1ccfe60..9f8b0c28 100644 --- a/webui/static/enrichment-manager.js +++ b/webui/static/enrichment-manager.js @@ -149,7 +149,7 @@ async function openEnrichmentManager(workerId) {