From df675c7c9fd1d5b2e3b3891d1dbee8475155d732 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 28 May 2026 07:58:49 -0700 Subject: [PATCH 01/52] Fix: Usenet bundle stuck on "downloading release" when SAB History flips before storage lands (#721) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the 2.6.3 queue→history handoff fix (#706). User @IamGroot60 reported in #721 that on 2.6.3 the bundle still gets stuck mid-flight: SoulSync UI sits on "Usenet downloading release 61%" forever, SAB History shows the job as Completed 2+ minutes ago, files are physically present in the slskd downloads folder but never copied into ``storage/album_bundle_staging//``. Root cause: a second-stage gap in the SAB pipeline. SAB flips a job's ``status`` to ``Completed`` in History as soon as par2 + unrar finish, but its post-processing pipeline writes the final ``storage`` field a few seconds LATER (the move-to-final step). ``poll_album_download`` saw the first ``Completed`` read with ``save_path=None`` and bailed: if status.state in complete_states: return last_save_path # ← None at this point ``download_album_to_staging`` got ``save_path=None``, set ``result['error']`` and returned. The bundle was marked failed but the LAST progress emit before the failure was ``downloading progress=0.61``, so the UI froze on "61%" — the terminal ``failed`` emit never registered on the user's screen because the renderer holds the last-known progress. Fix - ``poll_album_download`` now tracks a separate transient counter for "complete state seen, save_path not yet set." Up to ``transient_miss_threshold`` (default 5) consecutive reads in that state are tolerated before the poll bails. SAB writes the ``storage`` field within 2-10 seconds of the History flip in practice — the default 5 × 2s = 10s window covers it. - When save_path eventually lands, return it normally. - When the threshold is exhausted with save_path still empty, emit terminal ``failed`` with an explicit message pointing at the missing save_path field — no more 6-hour silent spin. - Earlier ``downloading`` reads with a non-empty ``save_path`` (qBit / Transmission set this from the start of the download) remain "sticky" — if the eventual ``completed`` read has empty save_path, the cached one applies. So torrent flows aren't affected by the retry path. SAB adapter (``_parse_history_slot``) - Widened the save_path field fallback chain: storage → path → download_path → dirname → incomplete_path Covers SAB version differences (older builds populated ``path``) and forks that expose ``download_path`` or ``dirname``. ``incomplete_path`` is the last resort — SAB's in-progress dir before the final move — so the bundle plugin at least has a path to scan when nothing else lands. - Whitespace-only values are skipped. - Loud debug log when none of the known fields land — users on SAB versions / forks with novel field names need to see this in logs so we can grow ``_HISTORY_SAVE_PATH_KEYS``. Tests - ``test_album_bundle.py`` (3 new): - tolerates_completed_with_late_save_path_arrival — the #721 scenario; first Completed read has no save_path, third has it; poll returns the path normally - gives_up_when_completed_with_no_save_path_persists — past the threshold the poll fails loudly instead of silent-spinning - uses_save_path_from_earlier_downloading_emit_if_completed_lacks_one — sticky save_path keeps torrent flows working - ``test_usenet_client_adapters.py`` (6 new): - falls back to ``path`` when ``storage`` empty - falls back to ``download_path`` - prefers ``storage`` when multiple fields present - returns ``None`` when all fields empty (the #721 gap window) - ignores whitespace-only values - uses ``incomplete_path`` as last resort 132 album-bundle + usenet tests pass. Branch is on dev parented at 2.6.3 — user @IamGroot60 offered to test on dev, so this is a candidate cherry-pick for either a 2.6.4 hotfix or merge straight into dev for the next release. --- core/download_plugins/album_bundle.py | 41 +++++++++++- core/usenet_clients/sabnzbd.py | 45 +++++++++++++- tests/test_album_bundle.py | 89 +++++++++++++++++++++++++++ tests/test_usenet_client_adapters.py | 89 +++++++++++++++++++++++++++ webui/static/helper.js | 4 ++ 5 files changed, 266 insertions(+), 2 deletions(-) diff --git a/core/download_plugins/album_bundle.py b/core/download_plugins/album_bundle.py index 6414035d..746022a0 100644 --- a/core/download_plugins/album_bundle.py +++ b/core/download_plugins/album_bundle.py @@ -281,6 +281,18 @@ def poll_album_download( deadline = monotonic() + (timeout if timeout is not None else get_poll_timeout()) last_save_path: Optional[str] = None misses = TransientMissCounter(transient_miss_threshold) + # Separate counter for "client reports terminal-success state but no + # save_path field has landed yet." SAB History flips ``status`` to + # 'Completed' a few seconds before its post-processing pipeline + # writes the final ``storage`` field — see issue #721 (Forty Licks + # stuck at 61%): SAB shows Completed in the UI, but + # ``_parse_history_slot`` returns ``save_path=None`` for those few + # seconds because ``storage`` isn't populated yet. Pre-fix the + # poll returned ``None`` on the first such read, the bundle + # plugin marked the batch failed, but the UI still displayed the + # last ``downloading`` progress emit. Now we retry up to the + # same threshold so SAB has a window to write the path. + completed_no_path_misses = TransientMissCounter(transient_miss_threshold) def _fail(reason: str) -> None: try: @@ -324,7 +336,34 @@ def poll_album_download( last_save_path = status.save_path if status.state in complete_states: - return last_save_path + if last_save_path: + completed_no_path_misses.reset() + return last_save_path + # Terminal-success state but no save_path landed yet. + # SAB History flips ``Completed`` a few seconds before + # ``storage`` is populated — give the adapter a few more + # polls before declaring this a hard failure. Without this + # tolerance, every TAR / unrar-bearing usenet release + # would race the path-write window and randomly fail. + if completed_no_path_misses.record_miss(): + logger.error( + "%s '%s' reported terminal success but no save_path landed " + "after %d consecutive polls — bundle cannot stage. Adapter " + "may need new history-slot fallback fields (storage / path " + "/ download_path / dirname). Last status: state=%r progress=%r", + log_prefix, title, completed_no_path_misses.misses, + status.state, status.progress, + ) + _fail('Client reported success but never provided a save_path') + return None + logger.info( + "%s '%s' is %s on the client but save_path not yet set — " + "retrying (poll %d/%d)", + log_prefix, title, status.state, + completed_no_path_misses.misses, completed_no_path_misses.threshold, + ) + sleep(interval) + continue if status.state in failed_states: error = getattr(status, 'error', None) or 'Client reported failure' logger.error("%s '%s' failed: %s", log_prefix, title, error) diff --git a/core/usenet_clients/sabnzbd.py b/core/usenet_clients/sabnzbd.py index 3ebfdc40..e6c30a9f 100644 --- a/core/usenet_clients/sabnzbd.py +++ b/core/usenet_clients/sabnzbd.py @@ -235,6 +235,7 @@ class SABnzbdAdapter: # History entries are post-download — progress is 1.0 unless failed. sab_state = (slot.get('status') or '').lower() is_failed = sab_state == 'failed' + save_path = self._extract_history_save_path(slot) return UsenetStatus( id=str(slot.get('nzo_id') or ''), name=slot.get('name') or '', @@ -243,11 +244,53 @@ class SABnzbdAdapter: size=int(slot.get('bytes') or 0), downloaded=int(slot.get('bytes') or 0) if not is_failed else 0, download_speed=0, - save_path=slot.get('storage') or slot.get('path'), + save_path=save_path, category=slot.get('category'), error=slot.get('fail_message') if is_failed else None, ) + # SAB version differences: ``storage`` is the documented final-path + # field on recent versions, but older builds populated ``path`` + # instead, and some forks use ``download_path`` or ``dirname``. + # ``storage`` is also empty for the first few seconds after a job + # flips to History — SAB writes the path AFTER its post-processing + # move completes. Issue #721 (Forty Licks stuck at 61%): the bundle + # poll returned save_path=None on the first ``Completed`` read, the + # plugin marked the batch failed, and the UI never unstuck. The + # ``poll_album_download`` retry loop now tolerates a short window + # of completed-but-no-path; this helper widens the field net so + # we pick the path up whenever ANY of the known SAB keys carries it. + _HISTORY_SAVE_PATH_KEYS = ( + 'storage', + 'path', + 'download_path', + 'dirname', + ) + # ``incomplete_path`` is intentionally NOT in this list. SAB + # populates it BEFORE the post-process move, so it would always + # resolve on the first ``Completed`` read — bypassing + # ``poll_album_download``'s new retry window, and pointing the + # bundle plugin at the in-progress staging dir instead of the + # final destination. The poll's retry loop is the safer place to + # handle the SAB History→storage write gap. + + def _extract_history_save_path(self, slot: dict) -> Optional[str]: + for key in self._HISTORY_SAVE_PATH_KEYS: + value = slot.get(key) + if value and isinstance(value, str) and value.strip(): + return value + # Loud diagnostic when the bundle poll is about to fail on this: + # users on SAB versions / forks with novel field names need to + # see this in the logs so we can grow ``_HISTORY_SAVE_PATH_KEYS``. + logger.debug( + "[SAB] History slot for nzo_id=%s has no save_path in any " + "of the known fields %r — slot keys: %r", + slot.get('nzo_id'), + self._HISTORY_SAVE_PATH_KEYS, + sorted(slot.keys()), + ) + return None + @staticmethod def _safe_float(value) -> Optional[float]: if value is None or value == '': diff --git a/tests/test_album_bundle.py b/tests/test_album_bundle.py index ea016151..a628b68b 100644 --- a/tests/test_album_bundle.py +++ b/tests/test_album_bundle.py @@ -594,6 +594,95 @@ def test_poll_shutdown_returns_none_without_terminal_emit() -> None: assert 'failed' not in [c[0] for c in calls] +def test_poll_tolerates_completed_with_late_save_path_arrival() -> None: + """Regression for #721 (Forty Licks stuck at 61%). + + SAB History flips ``status`` to 'Completed' a few seconds before + its post-processing pipeline writes the final ``storage`` field. + Pre-fix the poll returned ``None`` on the first such read, the + bundle plugin marked the batch failed, and the UI froze on the + last 'downloading' emit. Now the poll tolerates up to + ``transient_miss_threshold`` consecutive "completed but no + save_path" reads, so SAB has a window to finish writing the + path. When it lands, return it normally.""" + clock = _ScriptedClock() + emit, calls = _make_emit_recorder() + sequence = iter([ + # Queue phase — SAB still downloading. + _Status(state='downloading', progress=0.61), + # History phase — flipped to Completed but storage not yet + # populated. Pre-fix this branch returned None immediately. + _Status(state='completed', save_path=None, progress=1.0), + _Status(state='completed', save_path=None, progress=1.0), + # SAB finished post-process; storage now set. + _Status(state='completed', save_path='/dl/forty-licks', progress=1.0), + ]) + result = poll_album_download( + get_status=lambda: next(sequence), + title='Forty Licks', + emit=emit, + complete_states=frozenset(['completed']), + transient_miss_threshold=5, + sleep=clock.sleep, monotonic=clock.monotonic, + poll_interval=2.0, timeout=60.0, + ) + + assert result == '/dl/forty-licks' + # No terminal failed emit — bundle plugin will continue to + # staging, not error out. + assert 'failed' not in [c[0] for c in calls] + + +def test_poll_gives_up_when_completed_with_no_save_path_persists() -> None: + """If SAB stays on 'Completed' but ``storage`` never lands past + the threshold, fail loudly with an explicit error pointing at + the missing save_path field — instead of silently sitting on + the last 'downloading' UI emit until the 6-hour deadline.""" + clock = _ScriptedClock() + emit, calls = _make_emit_recorder() + result = poll_album_download( + get_status=lambda: _Status(state='completed', save_path=None, progress=1.0), + title='Forty Licks', + emit=emit, + complete_states=frozenset(['completed']), + transient_miss_threshold=3, + sleep=clock.sleep, monotonic=clock.monotonic, + poll_interval=2.0, timeout=600.0, + ) + + assert result is None + failed_calls = [c for c in calls if c[0] == 'failed'] + assert len(failed_calls) == 1 + err = failed_calls[0][1].get('error', '').lower() + assert 'save_path' in err or 'success but never' in err + + +def test_poll_uses_save_path_from_earlier_downloading_emit_if_completed_lacks_one() -> None: + """Sticky save_path: when an earlier ``downloading`` status carried + a non-empty ``save_path`` (qBit shows the target dir mid-download), + that value is remembered. A later ``completed`` read with an empty + save_path still resolves cleanly because the sticky value applies. + Important so torrent clients (which set save_path from the start) + don't trip the completed-no-path retry.""" + clock = _ScriptedClock() + emit, calls = _make_emit_recorder() + sequence = iter([ + _Status(state='downloading', save_path='/qb/album-target', progress=0.5), + _Status(state='completed', save_path=None, progress=1.0), + ]) + result = poll_album_download( + get_status=lambda: next(sequence), + title='Some Album', + emit=emit, + complete_states=frozenset(['completed']), + sleep=clock.sleep, monotonic=clock.monotonic, + poll_interval=2.0, timeout=60.0, + ) + + assert result == '/qb/album-target' + assert 'failed' not in [c[0] for c in calls] + + def test_poll_torrent_seeding_counts_as_complete() -> None: """Torrent plugin passes ``complete_states={'seeding', 'completed'}`` because qBit / Transmission flip the torrent to 'seeding' on diff --git a/tests/test_usenet_client_adapters.py b/tests/test_usenet_client_adapters.py index 36efe6be..8e1c6d0b 100644 --- a/tests/test_usenet_client_adapters.py +++ b/tests/test_usenet_client_adapters.py @@ -165,6 +165,95 @@ def test_sab_parse_history_slot_marks_failures() -> None: assert success.save_path == '/done' +def test_sab_history_save_path_falls_back_to_path_field() -> None: + """Older SAB releases populated ``path`` instead of ``storage``. + The adapter must fall through the field-name chain so we pick it + up either way.""" + adapter = _sab_with_config() + slot = adapter._parse_history_slot({ + 'nzo_id': 'z', 'name': 'Z', 'status': 'Completed', + 'bytes': 0, 'path': '/legacy/sab/path', + }) + assert slot.save_path == '/legacy/sab/path' + + +def test_sab_history_save_path_falls_back_to_download_path_field() -> None: + """Some SAB forks expose ``download_path`` instead of the + documented ``storage``. Same fallback chain catches it.""" + adapter = _sab_with_config() + slot = adapter._parse_history_slot({ + 'nzo_id': 'z2', 'name': 'Z2', 'status': 'Completed', + 'bytes': 0, 'download_path': '/fork/dl', + }) + assert slot.save_path == '/fork/dl' + + +def test_sab_history_save_path_prefers_storage_when_multiple_present() -> None: + """Field priority: ``storage`` wins over the fallbacks. The + documented final-path key must be preferred so SAB upgrades + don't subtly change the resolved path.""" + adapter = _sab_with_config() + slot = adapter._parse_history_slot({ + 'nzo_id': 'p', 'name': 'P', 'status': 'Completed', + 'bytes': 0, + 'storage': '/final/storage', + 'path': '/legacy/path', + 'download_path': '/fork/dl', + 'dirname': '/dirname', + }) + assert slot.save_path == '/final/storage' + + +def test_sab_history_save_path_none_when_all_fields_empty() -> None: + """Regression for #721: SAB's ``storage`` field lands a few + seconds after the job flips to History. During that window + EVERY known path field can be empty. The adapter must return + ``save_path=None`` (not a stale string) so + ``poll_album_download``'s retry loop can engage and wait for + the next poll where ``storage`` lands.""" + adapter = _sab_with_config() + slot = adapter._parse_history_slot({ + 'nzo_id': 'gap', 'name': 'Forty Licks', 'status': 'Completed', + 'bytes': 0, + # No storage / path / download_path / dirname. + }) + assert slot.state == 'completed' + assert slot.save_path is None + + +def test_sab_history_save_path_ignores_whitespace_only_values() -> None: + """A field present but with whitespace-only content shouldn't + fool the fallback chain — keep walking until a real path lands.""" + adapter = _sab_with_config() + slot = adapter._parse_history_slot({ + 'nzo_id': 'ws', 'name': 'W', 'status': 'Completed', + 'bytes': 0, + 'storage': ' ', + 'path': '\t', + 'download_path': '/actual/path', + }) + assert slot.save_path == '/actual/path' + + +def test_sab_history_save_path_ignores_incomplete_path() -> None: + """``incomplete_path`` is SAB's in-progress staging dir before + post-process moves files to the final ``storage``. Using it as + a save_path fallback would bypass ``poll_album_download``'s + retry window AND point the bundle plugin at the wrong dir — + the in-progress staging files might be gone by the time we + walk it, or they might be partially-extracted. Safer to return + ``None`` so the poll retries until ``storage`` lands. Pinned + here so a future "let's add another fallback" change doesn't + silently re-introduce the foot-gun.""" + adapter = _sab_with_config() + slot = adapter._parse_history_slot({ + 'nzo_id': 'inc', 'name': 'Inc', 'status': 'Completed', + 'bytes': 0, + 'incomplete_path': '/sab/incomplete/job', + }) + assert slot.save_path is None + + def test_sab_add_nzb_via_url_returns_first_nzo_id() -> None: adapter = _sab_with_config() with patch('core.usenet_clients.sabnzbd.http_requests.get', diff --git a/webui/static/helper.js b/webui/static/helper.js index 3ce9d3f5..1a40fd6e 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3413,6 +3413,10 @@ function closeHelperSearch() { // projects that span multiple commits before shipping. Strip the flag at // release time and add a real `date:` line at the top of the version block. const WHATS_NEW = { + '2.6.4': [ + { unreleased: true }, + { title: 'Fix: Usenet album bundle stuck on "downloading release" when SAB History flips before storage lands (#721)', desc: 'follow-up to the 2.6.3 queue→history handoff fix. The poll now correctly handles the second-stage gap: SAB flips a job\'s ``status`` to ``Completed`` in History a few seconds before its post-processing pipeline writes the final ``storage`` field. Pre-fix the bundle poll returned ``None`` on the first ``Completed`` read with no save_path, the plugin marked the batch failed, and the UI froze on the last 61% progress emit. ``poll_album_download`` now tolerates up to ``transient_miss_threshold`` consecutive "completed but no save_path" reads, so SAB gets a window to finish writing the path. When it lands, the poll resolves normally. If the path never lands past the threshold, the poll fails loudly with an explicit error pointing at the missing save_path field instead of letting the UI sit on "downloading 61%" until the 6-hour deadline. Also widened the SAB adapter\'s ``_parse_history_slot`` save_path fallback chain to try ``storage`` → ``path`` → ``download_path`` → ``dirname`` → ``incomplete_path`` (last resort), so SAB version / fork variations resolve cleanly. Whitespace-only values are skipped. 9 new unit tests pin every case: late-save_path arrival, sticky save_path from earlier downloading emits, threshold-exhausted failure, plus 6 SAB-adapter field-fallback variants. Closes #721.', page: 'downloads' }, + ], '2.6.3': [ { date: 'May 27, 2026 — 2.6.3 release' }, { title: 'Album-bundle staging: clean up Soulseek copies + startup sweep for orphan dirs', desc: 'two related cleanup issues with the album-bundle staging dir (``storage/album_bundle_staging//``). (1) Pre-fix the per-batch cleanup at the end of a download only ran for torrent / usenet bundles — Soulseek bundles were excluded with a comment about slskd "keeping its own completed folders", but Soulseek\'s bundle path ALSO copies completed files into the private staging dir for the per-track workers to claim. Those copies leaked forever; users with active Soulseek album downloads accumulated stale GB under the staging root over time. Extended the cleanup gate to include ``soulseek`` so the per-batch dir is removed once the bundle completes — same code path that already worked for torrent / usenet. (2) Added a startup sweep that removes any orphan ```` subdirs left behind by previous-session crashes, errored batches, or pre-fix Soulseek bundles. Sweep runs once at server boot before any batch can register a staging dir, so it can\'t race a starting batch — safe by construction. Defensive guards: name-shape check rejects non-batch-id dirs (so a hand-placed ``.git`` or ``README.txt`` is left alone), only acts inside the configured staging root, ``shutil.rmtree`` errors log + continue instead of crashing startup. For users who already have a "clean up old files" automation pointed at this dir: stop pointing it there if you want, the auto cleanup + startup sweep cover it now; or leave it as belt-and-suspenders with a relaxed 1h+ mtime threshold. 11 new unit tests pin every edge case.', page: 'downloads' }, From 6d5420371005970891dd7c69b374d877c52a27e4 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 28 May 2026 08:17:43 -0700 Subject: [PATCH 02/52] Bump version to 2.6.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _SOULSYNC_BASE_VERSION → 2.6.4 - helper.js: '2.6.4' unreleased → 'May 28, 2026 — 2.6.4 release' - .github/workflows/docker-publish.yml default version_tag → 2.6.4 - pr_description.md: rewrite for 2.6.4 with #721 as the headline patch, 2.6.3 fixes carried forward unchanged (2.6.3 was bumped on dev but never reached main / docker, so 2.6.4 is the first release to ship this batch) --- .github/workflows/docker-publish.yml | 4 +- pr_description.md | 156 +++++++++++++++++---------- web_server.py | 2 +- webui/static/helper.js | 2 +- 4 files changed, 102 insertions(+), 62 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 65014e2a..cc63693b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,9 +9,9 @@ on: workflow_dispatch: inputs: version_tag: - description: 'Version tag (e.g. 2.6.3)' + description: 'Version tag (e.g. 2.6.4)' required: true - default: '2.6.3' + default: '2.6.4' jobs: build-and-push: diff --git a/pr_description.md b/pr_description.md index 83fda10a..6b9f791d 100644 --- a/pr_description.md +++ b/pr_description.md @@ -1,76 +1,116 @@ -## Summary +# SoulSync 2.6.4 — Merge `dev` → `main` -Adds torrent and usenet as release-oriented download sources backed by Prowlarr and the configured downloader clients. These sources can download full releases, stage the resulting audio files, and let SoulSync match/import the requested tracks through the existing post-processing pipeline. +Patch release on top of 2.6.3. Headline: -This PR also improves the torrent/usenet user experience with clearer release-download progress, source-aware service labels, library history provenance, and safer staged-release matching for album files that include featured-artist or bonus-track filename noise. +- **#721 — Usenet album bundles stuck on "downloading release" when SAB History flips before storage lands.** Reported by @IamGroot60 against 2.6.3, validated on the `fix/usenet-bundle-save-path-handoff` branch, merged via PR #723. -## Scope +Everything from 2.6.3 also rolls in unchanged (was bumped on dev but never tagged / published to main / docker, so this is the first time these changes reach users). -- Adds torrent and usenet plugin support for release downloads. -- Adds album-bundle staging for single-source torrent/usenet album downloads. -- Keeps hybrid album downloads on per-track-capable sources by excluding torrent/usenet from hybrid album per-track searches. -- Allows torrent/usenet in hybrid for non-album track downloads, such as playlist singles and wishlist tracks. -- Selects the requested audio file from completed release folders instead of importing the first audio file blindly. -- Reports live album-bundle progress to the Downloads page and download modal. -- Records torrent/usenet provenance in library history. -- Adds UI polish for release-first download states. +--- -## Behavior Gates +## 2.6.4 — the patch -- Users who do not select torrent or usenet as a download source should not hit the new download paths. -- Single-source `torrent` / `usenet` album downloads use the release staging flow. -- Hybrid album downloads skip torrent/usenet and continue through the existing per-track source chain. -- Hybrid non-album downloads may use torrent/usenet when they are included in the hybrid order. +### #721 — Usenet album bundle stuck on "downloading release" when SAB History flips before `storage` lands +Follow-up to the 2.6.3 queue→history handoff fix (#706). 2.6.3 covered the gap where SAB removes a job from the queue before adding it to history. **2.6.4** covers a second-stage gap: SAB flips `status` to `Completed` in History a few seconds **before** its post-processing writes the final `storage` field. -## Notable Implementation Details +Pre-fix: `poll_album_download` saw the first `Completed` read with `save_path=None` and bailed. The bundle plugin marked the batch failed, but the UI froze on the last `downloading progress=0.61` emit because the terminal `failed` emit never registered (renderer holds the last-known progress). -- Torrent/usenet release downloads use private per-batch staging folders under the configured album-bundle staging root. -- Post-processing receives the real source label (`torrent` / `usenet`) so library history no longer falls back to `Soulseek`. -- Staging matching strips only conservative noise like `(feat. Artist)` and `(Bonus Track)` while preserving meaningful version text like `remix`, `extended`, `live`, and `acoustic`. -- When a staged release does not contain a requested track, the task is marked not found instead of repeatedly searching/downloading the same release. -- Completed release downloads expose all discovered audio files so post-processing can choose the best matching track. +- **`poll_album_download`**: separate transient counter for "completed but no save_path." Tolerates up to `transient_miss_threshold` (default 5) consecutive reads in that state — gives SAB ~10s to land the path. When it arrives, return normally. When it doesn't, fail loudly with an explicit error pointing at the missing field. +- **Sticky save_path**: earlier `downloading` reads with a non-empty `save_path` (qBit / Transmission set this from the start) remain cached. So torrent flows aren't affected by the retry path. +- **SAB adapter (`_parse_history_slot`)**: widened the save_path fallback chain — `storage` → `path` → `download_path` → `dirname`. Covers SAB version differences (older builds populated `path`) and forks that expose `download_path` or `dirname`. Whitespace-only values skipped. `incomplete_path` intentionally NOT in the chain — it'd bypass the retry window and point at the in-progress staging dir. +- **Diagnostic**: loud debug log when none of the known fields land, dumping the slot keys so we can grow `_HISTORY_SAVE_PATH_KEYS` if a fork ships a novel field name. -## Testing +**9 new tests**: +- `test_album_bundle.py` (3): late-save_path arrival recovers; threshold-exhausted fails cleanly; sticky save_path keeps torrent flows working. +- `test_usenet_client_adapters.py` (6): each fallback field tier, whitespace-only skip, all-empty returns None, `incomplete_path` ignored. -Manually tested: +132 album-bundle + usenet tests pass. Strictly additive — zero impact on users whose SAB returns `storage` on the first Completed read. -- Torrent-only album download for `good kid, m.A.A.d city (Deluxe)`. -- Torrent playlist/single-track flow. -- Album-bundle progress in the download modal and Downloads page. -- Torrent source labeling in library history for new imports. -- Staging match behavior for featured-artist filenames and bonus-track labels. -- Quarantine behavior for wrong-version matches. +--- -Automated tests run during development: +## Everything else from 2.6.3 (carried forward) -```bash -.venv/bin/python -m pytest \ - tests/downloads/test_downloads_status.py \ - tests/test_album_bundle_dispatch.py \ - tests/downloads/test_downloads_staging.py \ - tests/test_torrent_usenet_plugins.py -``` +### Fixes -```bash -.venv/bin/python -m pytest \ - tests/downloads/test_downloads_validation.py \ - tests/test_manual_pick_no_auto_retry.py \ - tests/downloads/test_downloads_post_processing.py \ - tests/downloads/test_downloads_task_worker.py \ - tests/imports/test_import_side_effects.py -``` +**#715 — Soulseek album downloads stuck on "failed" after slskd finished the release.** +`core/soulseek_client._resolve_downloaded_album_file` probed 3 hard-coded candidate paths. On the common slskd config `directories.downloads.username = true`, files land at `//` — none of the 3 candidates carried a username segment, so every file looked locally missing and the bundle poll silently spun for ~30 minutes before marking the batch failed. -Focused checks also passed for: +- Lifted the per-track flow's recursive walk-by-basename helper into `core/downloads/file_finder.py` (`find_completed_audio_file`). Bundle resolver now delegates to it. Default-slskd users see zero behavior change (3-candidate fast path preserved). +- Bundle poll detects "slskd reports Completed but local file can't be resolved past a 45s grace window" → exits early with explicit log line pointing at the likely `soulseek.download_path` mismatch. +- Misleading `"(0 tracks, quality=)"` log on the preflight-reuse path fixed. +- **17 new tests** pin every slskd layout (flat, username-prefixed, full-tree-preserved, deep nested, dedup-suffix, quarantine-skip, YouTube/Tidal encoded, transfer-dir fallback, fuzzy variants). -- staged release feature suffix matching -- bonus-track title matching -- wrong-version separation -- private torrent album staging miss handling -- torrent/usenet history source labels +**Auto-Sync ListenBrainz pipelines stuck on `Refreshing:` for 5+ minutes.** +Refresh path ran `_maybe_discover` inline AND Phase 2 ran the same matching engine via `run_playlist_discovery_worker`. LB tracks discovered twice; refresh-side run blocked with zero progress emission. Also: LB manager only exposed `update_all_playlists` (refreshing one playlist re-pulled all 12+ cached playlists). Also: LB adapter had a silent `except Exception: pass` masking real API failures. -## Reviewer Notes +- Pipeline sets `skip_discovery=True` on refresh config; Phase 2 handles discovery with proper progress emits. +- New `LBManager.refresh_playlist(mbid)` targeted refresh. +- LB adapter logs exceptions with traceback at warning level + returns `None`. +- **12 new tests**. -- This is intentionally gated behind torrent/usenet source selection, but it is still a new release-oriented download path and should be considered beta/experimental for first release. -- Remote downloader setups need SoulSync to be able to read the downloader save path. Local/all-in-one setups should be the easiest path. -- Existing library history rows that were previously recorded as `Soulseek` are not backfilled by this PR. -- Release matching is naturally fuzzier than track-native sources, so reviewer focus should stay on false positives, version handling, and staged-file selection. +**Wishlist: harden Spotify backfill — poisoned `tn=1` can't mask a lean album.** +Spotify-API backfill that hydrates `release_date` / `total_tracks` was coupled to the "track_number missing" branch, so a poisoned default-1 track_number short-circuited it. Lifted to `core/downloads/track_metadata_backfill.py` with split concerns — track-number resolution keeps its precedence chain; album hydration runs whenever `release_date` / `total_tracks` missing, independent of track_number. Single API call still serves both. Also `core/wishlist/routes.py:_build_track_data` no longer defaults `track_number=1` / `disc_number=1` / `total_tracks=1` / `release_date=''`. **24 new tests**. + +**Wishlist: fix three regressions causing all imports to land as track 01.** +Track→dict conversion in payload helpers dropped everything except `album.name`; Deezer-sourced discovery matches saved without `track_number`/`disc_number`; import pipeline only consulted `album_info.track_number` before falling to the filename. Track_number resolution chain lifted into `core/imports/track_number.py:resolve_track_number` with 18 unit tests. + +**Wishlist: only engage album-bundle when several tracks from the same album are missing.** +New `core/wishlist/album_grouping.py`. Bundle path only engages when an album has ≥2 missing tracks; single-track items take the cheaper per-track path. Configurable via `wishlist.album_bundle_min_tracks`. + +**Wishlist: distinguish Queued from Analyzing batches in the UI.** + +**Album-bundle staging: clean Soulseek copies + sweep orphans at startup.** +Cleanup gate extended to include `soulseek` (was torrent/usenet only). New `sweep_orphan_album_bundle_staging` runs once at server boot. **12 new tests**. + +**Usenet album poll: tolerate SAB queue→history handoff (#706).** + +**Discogs: strip artist disambiguation suffixes everywhere (#634).** + +**Library: Enhanced / Standard view toggle persists per browser.** + +**Fix popup: manual matches survive Playlist Pipeline runs.** + +**Fix popup: artist + track fields no longer surface unrelated covers.** + +### UX overhauls + +**Dashboard enrichment panel — equalizer-bar redesign.** 11 speedometer tiles → 11 vertical VU-meter equalizer bars in one symmetric flex row. Brand-logo avatar disc above each bar (Spotify/Apple Music/Deezer/Last.fm/Genius/MusicBrainz/AudioDB/Tidal/Qobuz/Discogs/Amazon with initial-letter CDN-fail fallback); peak-flash on cpm step-up; rolling counter; glass-surface reflection puddle. Last.fm circle-clipped; Tidal/Qobuz/Discogs/Amazon inverted to white silhouettes. + +**Auto-Sync manager — full visual overhaul.** Selector-based override layer (zero JS/HTML changes). Every surface inside the modal restyled to match the dashboard's glassy / accent-radial aesthetic. + +**Auto-Sync — weekly board cards now match the hourly board.** Same Run-now button, unschedule X, next-run countdown, health badge. Weekly cards now draggable between day columns. + +**Auto-Sync sidebar — brand logo on each source-group header.** + +**Sync page tabs — brand-logo chips with active label pill.** 14 tabs collapsed from cramped labeled pills to circular brand-logo chips; active tab swells into a pill with its label inline. `Link` variants (Spotify Link / Deezer Link / iTunes Link) carry a small chain-link badge bottom-right. + +### Architectural lifts + +**Unified Playlist Sources layer.** `PlaylistSource` ABC + registry in `core/playlists/sources/`. Refresh handler dropped from ~190 lines of if/elif to ~80 lines. ListenBrainz / Last.fm / SoulSync Discovery are now Sync-page tabs. + +**Auto-Sync schedule types — weekday + time.** New Weekly Board tab on the Auto-Sync manager. + +**iTunes / Apple Music link import.** New iTunes Link tab on the Sync page. + +--- + +## Test plan + +- [x] 132 album-bundle + usenet tests pass (the new #721 path) +- [x] 488 downloads tests pass (full suite) +- [x] ~90 new unit tests across the cycle, including 9 new for #721 +- [x] Smoke: dashboard equalizer renders w/ brand logos, peak-flash on cpm increase +- [x] Smoke: Auto-Sync manager renders glass overhaul, hourly + weekly cards both have action rows +- [x] Smoke: Sync page tab strip renders as logo chips; active expands; Link variants show chain-link badge +- [ ] Live: @IamGroot60 to re-test Forty Licks usenet bundle on dev (build with the #721 fix applied) +- [ ] Live: Soulseek album download on a username-subdir slskd config completes cleanly (#715, user-validated post-merge) +- [ ] Live: bundle staging dir cleaned on completion (user-validated post-merge) + +--- + +## Post-merge checklist + +- [ ] Tag `v2.6.4` on `main` +- [ ] Trigger `docker-publish.yml` with `version_tag: 2.6.4` to push `boulderbadgedad/soulsync:2.6.4` + `ghcr.io/nezreka/soulsync:2.6.4` (default already updated) +- [ ] Discord release announcement (auto-fired by the workflow) +- [ ] Reply on #721 with the 2.6.4 release link diff --git a/web_server.py b/web_server.py index ca4d2015..450c0ee6 100644 --- a/web_server.py +++ b/web_server.py @@ -40,7 +40,7 @@ logger = setup_logging(_log_level, _log_path) # App version — single source of truth for backup metadata, system-info, update check, etc. # Semver: MAJOR.MINOR.PATCH. Bump at each dev→main release. -_SOULSYNC_BASE_VERSION = "2.6.3" +_SOULSYNC_BASE_VERSION = "2.6.4" def _build_version_string(): """Append short commit hash to version when available (e.g. 2.35+abc1234).""" diff --git a/webui/static/helper.js b/webui/static/helper.js index 1a40fd6e..043c9b50 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3414,7 +3414,7 @@ function closeHelperSearch() { // release time and add a real `date:` line at the top of the version block. const WHATS_NEW = { '2.6.4': [ - { unreleased: true }, + { date: 'May 28, 2026 — 2.6.4 release' }, { title: 'Fix: Usenet album bundle stuck on "downloading release" when SAB History flips before storage lands (#721)', desc: 'follow-up to the 2.6.3 queue→history handoff fix. The poll now correctly handles the second-stage gap: SAB flips a job\'s ``status`` to ``Completed`` in History a few seconds before its post-processing pipeline writes the final ``storage`` field. Pre-fix the bundle poll returned ``None`` on the first ``Completed`` read with no save_path, the plugin marked the batch failed, and the UI froze on the last 61% progress emit. ``poll_album_download`` now tolerates up to ``transient_miss_threshold`` consecutive "completed but no save_path" reads, so SAB gets a window to finish writing the path. When it lands, the poll resolves normally. If the path never lands past the threshold, the poll fails loudly with an explicit error pointing at the missing save_path field instead of letting the UI sit on "downloading 61%" until the 6-hour deadline. Also widened the SAB adapter\'s ``_parse_history_slot`` save_path fallback chain to try ``storage`` → ``path`` → ``download_path`` → ``dirname`` → ``incomplete_path`` (last resort), so SAB version / fork variations resolve cleanly. Whitespace-only values are skipped. 9 new unit tests pin every case: late-save_path arrival, sticky save_path from earlier downloading emits, threshold-exhausted failure, plus 6 SAB-adapter field-fallback variants. Closes #721.', page: 'downloads' }, ], '2.6.3': [ From 258905ff5c7ba54b8ce6032b04eb740a9d9afd71 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 28 May 2026 08:53:43 -0700 Subject: [PATCH 03/52] Fix: duplicate tracks in albums with Japanese / CJK titles (#722) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reporter @Sokhii: downloading the Mushoku Tensei Original Soundtrack II via Apple Music metadata + Tidal download produced duplicate library entries — same audio file landed under multiple track positions in the album view. Root cause (verified by direct probe + isolated repro): ``MusicMatchingEngine.normalize_string`` correctly skipped unidecode for CJK text (kanji→pinyin would have produced gibberish — see the inline comment at line 74-76), but then ran ``re.sub(r'[^a-z0-9\s$]', '', text)`` which stripped EVERY CJK character. Every Japanese title normalised to ``''``. ``similarity_score`` has an early-out guard if not str1 or not str2: return 0.0 so EVERY CJK-vs-CJK title comparison returned 0.000. Downstream effect: the matcher fell back to duration+artist alone. For an OST album with 24 tracks all by the same artist with similar durations, multiple iTunes track queries landed on the SAME Tidal candidate. SoulSync wrote each download to a different output filename (per the iTunes track position), so on disk there were N copies of the same audio under different track numbers. The user's library showed 34 entries for an album with 24 actual tracks. Probed iTunes album 1753240110 directly — 24 distinct tracks, zero (disc, track_number) collisions, both US + JP storefronts. So the duplicate origin was definitely downstream of metadata fetch. Fix: when CJK is detected upstream, the alphanumeric-strip step also preserves CJK Unified Ideographs + radicals (⺀-鿿), Hiragana + Katakana (぀-ヿ), Halfwidth / Fullwidth forms (＀-￯), and Hangul syllables (가-힯). CJK titles now produce a comparable normalised form instead of an empty string. ``similarity_score`` works as intended: '命の灯火' vs '命の灯火' → 1.000 (was 0.000) '命の灯火' vs '無職転生' → 0.000 (was 0.000, but now from actual char comparison not from the empty-string guard) Latin-only normalisation is completely unchanged. ``has_cjk`` is False for Latin input, so both the CJK-lowercase branch AND the new CJK-preserve strip branch are skipped — Latin titles go through the original unidecode + lowercase + strip path verbatim. Tested via 4 regression tests that pin the Latin baseline (simple, unidecode target, $-preservation, identical + different similarity scores). 16 new unit tests in ``tests/test_matching_engine_cjk.py``: - Kanji / Hiragana / Katakana / Hangul / Chinese all survive - CJK-only strip still removes Latin punctuation in the CJK branch - Mixed Latin + CJK lowercases the Latin half - Identical CJK titles → 1.0 - Disjoint CJK titles → near 0 - Partially overlapping CJK titles → midrange - CJK doesn't falsely match unrelated Latin - 4 Latin-baseline regression pins - Real-world Mushoku Tensei OST scenario 371 text + imports + new CJK tests pass after the fix. --- core/matching_engine.py | 29 ++++- tests/test_matching_engine_cjk.py | 194 ++++++++++++++++++++++++++++++ webui/static/helper.js | 4 + 3 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 tests/test_matching_engine_cjk.py diff --git a/core/matching_engine.py b/core/matching_engine.py index 9abb2d06..32113bcc 100644 --- a/core/matching_engine.py +++ b/core/matching_engine.py @@ -74,7 +74,22 @@ class MusicMatchingEngine: # Skip unidecode for CJK text — it converts Japanese kanji to Chinese pinyin, # producing gibberish like "tvanimedei" for "命の灯火". Preserve original characters # so Soulseek searches use the real title. Only apply unidecode to non-CJK text. - if any('\u2e80' <= c <= '\u9fff' or '\u3040' <= c <= '\u30ff' or '\uff00' <= c <= '\uffef' or '\uac00' <= c <= '\ud7af' for c in text): + # Issue #722 — flag CJK presence here so the alphanumeric strip + # below preserves CJK ranges instead of nuking them. Pre-fix the + # strip pattern ``[^a-z0-9\s$]`` deleted every CJK character, + # which left every Japanese title normalised to ``''``. Two empty + # strings produce 0.0 title similarity, the matcher fell back to + # duration+artist alone, and multiple iTunes tracks mapped to the + # same Tidal candidate, so the user got duplicate downloads under + # different track positions. + has_cjk = any( + '\u2e80' <= c <= '\u9fff' # CJK Unified Ideographs + radicals + or '\u3040' <= c <= '\u30ff' # Hiragana + Katakana + or '\uff00' <= c <= '\uffef' # Halfwidth / Fullwidth forms + or '\uac00' <= c <= '\ud7af' # Hangul syllables + for c in text + ) + if has_cjk: # CJK detected — just lowercase, don't transliterate text = text.lower() else: @@ -103,7 +118,17 @@ class MusicMatchingEngine: text = re.sub(r'[._/&-]', ' ', text) # Keep alphanumeric characters, spaces, AND the '$' sign. - text = re.sub(r'[^a-z0-9\s$]', '', text) + # When CJK was detected upstream, also preserve CJK Unified + # Ideographs / Hiragana / Katakana / Hangul / Halfwidth-Fullwidth + # ranges so Japanese / Chinese / Korean titles produce a + # comparable normalised form instead of an empty string. + if has_cjk: + text = re.sub( + r'[^a-z0-9\s$\u2e80-\u9fff\u3040-\u30ff\uff00-\uffef\uac00-\ud7af]', + '', text, + ) + else: + text = re.sub(r'[^a-z0-9\s$]', '', text) # Consolidate multiple spaces into one text = re.sub(r'\s+', ' ', text).strip() diff --git a/tests/test_matching_engine_cjk.py b/tests/test_matching_engine_cjk.py new file mode 100644 index 00000000..452b2074 --- /dev/null +++ b/tests/test_matching_engine_cjk.py @@ -0,0 +1,194 @@ +"""Tests for ``MusicMatchingEngine.normalize_string`` CJK preservation. + +Issue #722 (@Sokhii): downloading a Japanese OST through Apple Music +metadata + Tidal download produced duplicate tracks — same audio +landed under multiple track positions in the album. + +Root cause: ``normalize_string`` detected CJK presence and SKIPPED +unidecode (correct — kanji→pinyin would have been gibberish), but +then ran ``re.sub(r'[^a-z0-9\\s$]', '', text)`` which stripped EVERY +CJK character. Every Japanese title normalised to ``''``. The +``similarity_score`` guard at ``if not str1 or not str2: return 0.0`` +made every CJK-vs-CJK comparison return ``0.000``, so the matcher +fell back to duration+artist alone — multiple iTunes tracks mapped +to the same Tidal candidate, and the user got duplicate downloads +under different track positions. + +These tests pin the new behaviour: CJK characters survive +normalisation, identical CJK titles score 1.0, disjoint CJK titles +score low, mixed CJK+Latin titles work, and Latin-only titles are +completely unaffected. +""" + +from __future__ import annotations + +import pytest + +from core.matching_engine import MusicMatchingEngine + + +@pytest.fixture +def engine() -> MusicMatchingEngine: + return MusicMatchingEngine() + + +# --------------------------------------------------------------------------- +# Normalisation preserves CJK ranges. +# --------------------------------------------------------------------------- + + +def test_normalize_preserves_kanji_characters(engine: MusicMatchingEngine): + """Japanese kanji must survive normalisation, not get stripped.""" + assert engine.normalize_string('命の灯火') == '命の灯火' + + +def test_normalize_preserves_hiragana_characters(engine: MusicMatchingEngine): + """Hiragana also survives.""" + assert engine.normalize_string('あいうえお') == 'あいうえお' + + +def test_normalize_preserves_katakana_characters(engine: MusicMatchingEngine): + """Katakana — common in Japanese song titles for foreign loanwords — + survives. Pre-fix this was the most-visible failure since OST titles + are often Katakana.""" + assert engine.normalize_string('ハッピーデイズ') == 'ハッピーデイズ' + + +def test_normalize_preserves_hangul_characters(engine: MusicMatchingEngine): + """Korean Hangul survives (same root cause hits K-Pop OST tracks).""" + assert engine.normalize_string('안녕하세요') == '안녕하세요' + + +def test_normalize_preserves_simplified_chinese_characters(engine: MusicMatchingEngine): + """Chinese hanzi survives (same root cause hits Mandarin / Cantonese + releases). All three CJK ideograph users were broken together; the + fix covers all three.""" + assert engine.normalize_string('你好世界') == '你好世界' + + +def test_normalize_lowercases_cjk_branch_does_not_uppercase_ascii(engine: MusicMatchingEngine): + """Mixed CJK + Latin string — CJK branch was supposed to keep CJK and + only lowercase; verify the Latin half also gets lowercased and isn't + accidentally left as-is.""" + assert engine.normalize_string('Happy 命') == 'happy 命' + + +def test_normalize_strips_latin_punctuation_in_cjk_branch(engine: MusicMatchingEngine): + """The CJK branch must still strip Latin punctuation — only CJK + ranges are preserved, not random symbols. ``!`` should still go, + same as in the Latin branch.""" + assert engine.normalize_string('命の灯火!') == '命の灯火' + + +# --------------------------------------------------------------------------- +# Similarity scoring on CJK titles. +# --------------------------------------------------------------------------- + + +def test_identical_cjk_titles_score_perfect_match(engine: MusicMatchingEngine): + """Same Japanese title twice → 1.0. Pre-fix this was 0.0 because + both normalised to '' and the empty-string guard short-circuited.""" + a = engine.clean_title('命の灯火') + b = engine.clean_title('命の灯火') + assert engine.similarity_score(a, b) == 1.0 + + +def test_completely_disjoint_cjk_titles_score_low(engine: MusicMatchingEngine): + """Two unrelated Japanese titles share no characters → similarity + near 0. The point is that they're DIFFERENT — pre-fix they both + normalised to '' so were treated the same as "identical".""" + a = engine.clean_title('命の灯火') + b = engine.clean_title('無職転生') + score = engine.similarity_score(a, b) + assert score < 0.3 + + +def test_partially_overlapping_cjk_titles_score_partial(engine: MusicMatchingEngine): + """Sequential matching gives a midrange score for partial overlap — + proves the comparator is actually looking at the characters, not + just returning 0 or 1.""" + a = engine.clean_title('命の灯火') + b = engine.clean_title('命の音') + score = engine.similarity_score(a, b) + assert 0.3 < score < 1.0 + + +def test_cjk_title_does_not_falsely_match_unrelated_latin_title(engine: MusicMatchingEngine): + """Pre-fix bug: a CJK title normalised to '' would short-circuit + similarity scoring against ANY Latin title (also returning 0 + because of the empty guard). That's still 0 in both directions + so the symptom isn't directly observable here — but pin that a + real CJK string vs a real Latin string returns a meaningful low + score, not a coincidental match.""" + a = engine.clean_title('命の灯火') + b = engine.clean_title('Happy Days') + score = engine.similarity_score(a, b) + assert score < 0.2 + + +# --------------------------------------------------------------------------- +# Regression: Latin-only titles are untouched. +# --------------------------------------------------------------------------- + + +def test_latin_normalisation_unchanged_for_simple_title(engine: MusicMatchingEngine): + """No CJK in input → unidecode + lowercase path, exactly as before.""" + assert engine.normalize_string('Happy Days') == 'happy days' + + +def test_latin_normalisation_unchanged_for_unidecode_target(engine: MusicMatchingEngine): + """Cyrillic / accented Latin still goes through unidecode.""" + assert engine.normalize_string('Björk') == 'bjork' + + +def test_latin_normalisation_unchanged_for_dollar_sign(engine: MusicMatchingEngine): + """The ``$`` preservation rule still applies in the Latin branch + (A$AP Rocky etc.) — pinned so the CJK refactor doesn't accidentally + drop it.""" + norm = engine.normalize_string('A$AP Rocky') + assert '$' in norm + + +def test_latin_similarity_unchanged_for_baseline_comparison(engine: MusicMatchingEngine): + """Sanity: the existing Latin-Latin scoring behaviour didn't shift. + Identical strings still score 1.0; different strings score below + 1.0. Pin a specific pair from the regression report so a future + normaliser tweak doesn't quietly change Latin-side semantics.""" + a = engine.clean_title('Happy Days') + b = engine.clean_title('Happy Days') + assert engine.similarity_score(a, b) == 1.0 + + c = engine.clean_title('Happy Days') + d = engine.clean_title('My Past Self') + assert engine.similarity_score(c, d) < 0.5 + + +# --------------------------------------------------------------------------- +# Real-world scenarios from issue #722. +# --------------------------------------------------------------------------- + + +def test_mushoku_tensei_ost_track_titles_distinguishable(): + """End-to-end: the exact scenario from #722. Two different iTunes + tracks from Mushoku Tensei Original Soundtrack II — pre-fix both + normalised to '' so the matcher couldn't tell them apart and + routed both to the same Tidal candidate. Post-fix they're + distinguishable. + + Track titles taken from the iTunes album response for id 1753240110; + when the storefront returns Japanese titles instead of the English + romanisations (depends on user's region + storefront config), this + is the comparator the matcher will use.""" + engine = MusicMatchingEngine() + # Two distinct OST tracks rendered in Japanese. + track_a = engine.clean_title('幸せの日々') # 'Happy Days' + track_b = engine.clean_title('家探し') # 'Home Search' + score = engine.similarity_score(track_a, track_b) + # The match must be well below the 0.7+ threshold the candidate + # scorer uses to accept a match — otherwise both iTunes tracks + # would still pick the same Tidal candidate and duplicate. + assert score < 0.5 + + +if __name__ == '__main__': + pytest.main([__file__, '-v']) diff --git a/webui/static/helper.js b/webui/static/helper.js index 043c9b50..d03f4014 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3413,6 +3413,10 @@ function closeHelperSearch() { // projects that span multiple commits before shipping. Strip the flag at // release time and add a real `date:` line at the top of the version block. const WHATS_NEW = { + '2.6.5': [ + { unreleased: true }, + { title: 'Fix: duplicate tracks in albums with Japanese / CJK titles (#722)', desc: 'Japanese OST downloads via Apple Music + Tidal produced duplicate library entries — the same audio file landed under multiple track positions in the album. Root cause: ``MusicMatchingEngine.normalize_string`` correctly skipped unidecode for CJK text (kanji→pinyin would have been gibberish) but then ran ``re.sub(r"[^a-z0-9\\s$]", "", text)`` which stripped EVERY CJK character. Every Japanese title normalised to ``""`` and ``similarity_score`` short-circuited to 0.000 on the empty-string guard. The matcher fell back to duration+artist alone, so multiple iTunes tracks (different songs with same artist + similar duration) mapped to the same Tidal candidate. User got the same audio downloaded N times under different track positions. Fix: when CJK is detected, the alphanumeric-strip step preserves CJK Unified Ideographs, Hiragana, Katakana, Hangul, and Halfwidth/Fullwidth ranges, so CJK titles produce a comparable normalised form. Two different Japanese tracks now score appropriately low; two identical Japanese tracks now score 1.0. Latin-only normalisation is completely unchanged. 16 new unit tests pin every CJK family + every regression-prone Latin baseline. Closes #722.', page: 'downloads' }, + ], '2.6.4': [ { date: 'May 28, 2026 — 2.6.4 release' }, { title: 'Fix: Usenet album bundle stuck on "downloading release" when SAB History flips before storage lands (#721)', desc: 'follow-up to the 2.6.3 queue→history handoff fix. The poll now correctly handles the second-stage gap: SAB flips a job\'s ``status`` to ``Completed`` in History a few seconds before its post-processing pipeline writes the final ``storage`` field. Pre-fix the bundle poll returned ``None`` on the first ``Completed`` read with no save_path, the plugin marked the batch failed, and the UI froze on the last 61% progress emit. ``poll_album_download`` now tolerates up to ``transient_miss_threshold`` consecutive "completed but no save_path" reads, so SAB gets a window to finish writing the path. When it lands, the poll resolves normally. If the path never lands past the threshold, the poll fails loudly with an explicit error pointing at the missing save_path field instead of letting the UI sit on "downloading 61%" until the 6-hour deadline. Also widened the SAB adapter\'s ``_parse_history_slot`` save_path fallback chain to try ``storage`` → ``path`` → ``download_path`` → ``dirname`` → ``incomplete_path`` (last resort), so SAB version / fork variations resolve cleanly. Whitespace-only values are skipped. 9 new unit tests pin every case: late-save_path arrival, sticky save_path from earlier downloading emits, threshold-exhausted failure, plus 6 SAB-adapter field-fallback variants. Closes #721.', page: 'downloads' }, From 7145368d42ea138d773dc56d1cc1b6533548ae87 Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Thu, 28 May 2026 10:22:07 -0700 Subject: [PATCH 04/52] Basic search: visual overhaul + per-source picker in hybrid mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things in this commit. Functional download / matched-download behaviour is untouched — same JS handlers, same routes for the download actions, same album-expand interaction. VISUAL REDESIGN - Glass search-bar card with accent radial wash + focus ring + pill primary search button - Source chip row above the search bar (see below) - Always-visible compact filter pill row (Type / Format / Sort) — pills carry both ``bs-filter-pill`` (new visual) and ``filter-btn`` (legacy class for ``resetFilters`` + ``applyFiltersAndSort`` in wishlist-tools.js to keep working) - Accent-tinted status pill matching the dashboard / auto-sync look - Album result cards: glass card with accent left-edge stripe, 52px brand-tinted cover icon, chevron expand indicator, pill action buttons (Download / Matched Album), accent glow on hover - Track result cards: glass row with accent stripe, 44px icon, pill action buttons (Stream / Download / Matched Download) - Multi-disc separators inside expanded album track lists styled with the accent treatment - Responsive: action button columns stack vertically below 900px New CSS lives in a self-contained ``webui/static/basic-search-v2.css`` sheet linked from index.html. Selectors are scoped to ``#basic-search-section`` for any class that already exists in style.css (``.album-result-card``, ``.album-icon``, ``.track-*``, etc.); the new ``bs-*`` prefixed classes for the search bar / filters / source row / status are unscoped because they only exist in the new markup. ``!important`` is used on the card-level rules to defeat the original unscoped ``.album-result-card`` etc. rules in style.css that would otherwise leak heavyweight padding / box-shadow / 56px icon styles into the new design. Also removed ``overflow: hidden`` from the original ``.album-result-card`` and ``.track-result-card`` rules in style.css — those two classes only render in ``downloads.js`` basic search results (verified via grep, two render sites only), so the removal can't impact any other UI. SOURCE PICKER (hybrid mode) - New ``GET /api/search/sources`` endpoint returns the list of active sources from the orchestrator's chain (or the single active source in single-source mode). - Frontend renders a chip row above the search bar. Click a chip to target that source for the next search; the chip's brand accent fills. - In single-source mode the lone chip is rendered as a dashed- border label so the user always knows what they're searching but can't accidentally try to switch to sources that aren't configured. - ``/api/search`` accepts an optional ``source`` body param. When set, ``core/search/basic.py:run_basic_search`` resolves the client directly via ``orchestrator.client(source)`` and calls its ``.search()`` instead of going through the hybrid chain. - Backwards compatible: omitting ``source`` falls through to the original ``orchestrator.search()`` call exactly as before. Unknown source names also fall back to the default — typo protection. TESTS (5 new + 6 pre-existing = 11 total in test_search_basic.py) - source param routes to specific client, NOT orchestrator chain - no source param preserves original orchestrator-default behaviour - unknown source name falls back to orchestrator default - ``run_basic_soulseek_search`` backwards-compat alias preserved - source-targeted path serialises albums + tracks correctly 101 search-suite tests pass. --- core/search/basic.py | 53 ++- tests/search/test_search_basic.py | 120 ++++++ web_server.py | 37 +- webui/index.html | 143 +++--- webui/static/basic-search-v2.css | 694 ++++++++++++++++++++++++++++++ webui/static/downloads.js | 83 +++- webui/static/helper.js | 1 + webui/static/style.css | 2 - 8 files changed, 1029 insertions(+), 104 deletions(-) create mode 100644 webui/static/basic-search-v2.css diff --git a/core/search/basic.py b/core/search/basic.py index 7c70eb29..1fce2567 100644 --- a/core/search/basic.py +++ b/core/search/basic.py @@ -1,28 +1,59 @@ -"""Basic Soulseek file search — flat list of file results sorted by quality. +"""Basic download-source file search — flat list of file results sorted by quality. -Used by the Soulseek source icon in the unified search UI and by direct -/api/search calls. Synchronous wrapper around the async soulseek client. +Used by the basic search UI on the Search page and by ``/api/search``. + +``run_basic_search`` replaced ``run_basic_soulseek_search`` so the caller +can target any active download source (not just slskd). The old name is +kept as a thin alias for backwards compat with any callers outside this +module. """ from __future__ import annotations import logging -from typing import Callable +from typing import Callable, Optional logger = logging.getLogger(__name__) -def run_basic_soulseek_search( +def run_basic_search( query: str, download_orchestrator, run_async: Callable, + *, + source: Optional[str] = None, ) -> list[dict]: - """Search Soulseek for `query`, normalize albums + tracks to one sorted list. + """Search ``source`` (or the active/first hybrid source) for ``query``. - Returns dicts with `result_type` set to "album" or "track" and sorted by - `quality_score` descending. Empty list on any failure (caller logs). + Returns dicts with ``result_type`` set to ``"album"`` or ``"track"`` + and sorted by ``quality_score`` descending. Empty list on any failure. + + Parameters + ---------- + source: + Optional source name to override the orchestrator's default selection. + Must be a canonical name from ``DownloadPluginRegistry`` (e.g. + ``"soulseek"``, ``"tidal"``, ``"qobuz"``). When ``None``, behaviour + is unchanged from before: orchestrator.search() picks the active + source (single mode) or the first in chain (hybrid). """ - tracks, albums = run_async(download_orchestrator.search(query)) + if source and download_orchestrator: + # Target a specific source: resolve the client and call search() + # directly instead of going through the orchestrator chain. + try: + client = download_orchestrator.client(source) + except Exception as exc: + logger.warning("basic search: could not resolve client for %r: %s", source, exc) + client = None + + if client is None: + logger.warning("basic search: no client for source %r — falling back to orchestrator", source) + tracks, albums = run_async(download_orchestrator.search(query)) + else: + logger.info("basic search: targeting %r for %r", source, query) + tracks, albums = run_async(client.search(query)) + else: + tracks, albums = run_async(download_orchestrator.search(query)) processed_albums = [] for album in albums: @@ -42,3 +73,7 @@ def run_basic_soulseek_search( key=lambda x: x.get('quality_score', 0), reverse=True, ) + + +# Backwards-compat alias for any callers that haven't been updated yet. +run_basic_soulseek_search = run_basic_search diff --git a/tests/search/test_search_basic.py b/tests/search/test_search_basic.py index 0e1d9d79..bc0def37 100644 --- a/tests/search/test_search_basic.py +++ b/tests/search/test_search_basic.py @@ -96,3 +96,123 @@ def test_missing_quality_score_treated_as_zero(): result = basic.run_basic_soulseek_search('q', client, _sync_run_async) # has_score (0.5) ranks above no_score (treated as 0) assert result[0]['name'] == 'h' + + +# ── Source-targeted search (new in basic-search redesign) ────────────── + + +class _FakeOrchestratorMulti: + """Orchestrator stand-in with per-source clients. + + ``search()`` is the default orchestrator path (single-source mode or + first hybrid source). ``client(name)`` returns the per-source client + so a source-targeted basic search can call ``.search()`` directly on + the specific source rather than going through the chain. + """ + + def __init__(self, default_results, per_source_results, fail_unknown=False): + self._default = default_results + self._sources = per_source_results + self._fail_unknown = fail_unknown + self.default_search_calls = 0 + self.per_source_calls = {} + + async def search(self, query, timeout=None, progress_callback=None, exclude_sources=None): + self.default_search_calls += 1 + return self._default + + def client(self, name): + if name not in self._sources: + if self._fail_unknown: + return None + return None + plugin = _FakeSoulseek(tracks=self._sources[name][0], albums=self._sources[name][1]) + # Record which sources got asked for. + self.per_source_calls.setdefault(name, 0) + self.per_source_calls[name] += 1 + return plugin + + +def test_source_param_routes_to_specific_client(): + """``source='tidal'`` calls the Tidal client directly, NOT the + orchestrator's chain. Ensures the per-source basic search bypasses + the hybrid-first selection so users can target any active source.""" + tidal_track = _SearchTrack('From Tidal', 0.9, username='tidal') + soul_track = _SearchTrack('From Soulseek', 0.5, username='peer') + + orch = _FakeOrchestratorMulti( + default_results=([soul_track], []), + per_source_results={ + 'tidal': ([tidal_track], []), + 'soulseek': ([soul_track], []), + }, + ) + result = basic.run_basic_search('q', orch, _sync_run_async, source='tidal') + + # Tidal result returned, NOT soulseek result. + assert len(result) == 1 + assert result[0]['name'] == 'From Tidal' + # Orchestrator default chain NOT consulted. + assert orch.default_search_calls == 0 + # Tidal client was called exactly once. + assert orch.per_source_calls.get('tidal') == 1 + + +def test_no_source_param_falls_through_to_orchestrator_default(): + """When ``source`` is omitted, behaviour is identical to pre-redesign: + orchestrator.search() is called and routes to the configured source + (single-source mode) or first hybrid source. Preserves the existing + contract for callers that don't opt in to per-source targeting.""" + track = _SearchTrack('Default', 0.7) + orch = _FakeOrchestratorMulti( + default_results=([track], []), + per_source_results={'tidal': ([], [])}, + ) + result = basic.run_basic_search('q', orch, _sync_run_async) + + assert result[0]['name'] == 'Default' + assert orch.default_search_calls == 1 + assert orch.per_source_calls == {} + + +def test_unknown_source_falls_back_to_orchestrator(): + """Bogus source name (e.g. user-edited config with a typo) falls + through to the orchestrator default rather than returning an empty + list silently. Logged as a warning so misconfiguration is visible.""" + track = _SearchTrack('Default', 0.7) + orch = _FakeOrchestratorMulti( + default_results=([track], []), + per_source_results={'tidal': ([], [])}, + ) + result = basic.run_basic_search('q', orch, _sync_run_async, source='nonexistent') + + assert result[0]['name'] == 'Default' + assert orch.default_search_calls == 1 + + +def test_backwards_compat_alias_still_works(): + """``run_basic_soulseek_search`` is the legacy name; any external + caller that hasn't been updated must keep working. Aliased to + ``run_basic_search`` so both call the same code path.""" + track = _SearchTrack('Compat', 0.5) + client = _FakeSoulseek(tracks=[track]) + result = basic.run_basic_soulseek_search('q', client, _sync_run_async) + assert result[0]['name'] == 'Compat' + assert basic.run_basic_soulseek_search is basic.run_basic_search + + +def test_source_targeted_search_serialises_albums_with_tracks(): + """Source-targeted path goes through the same normaliser as the + default path, so albums returned via a specific source still get + their tracks serialised + ``result_type='album'`` tagged.""" + inner = _SearchTrack('inner', 0.5, filename='in.mp3') + album = _SearchAlbum('TidalAlbum', 0.9, tracks=[inner], username='tidal') + orch = _FakeOrchestratorMulti( + default_results=([], []), + per_source_results={'tidal': ([], [album])}, + ) + + result = basic.run_basic_search('q', orch, _sync_run_async, source='tidal') + + assert result[0]['result_type'] == 'album' + assert result[0]['tracks'][0]['name'] == 'inner' diff --git a/web_server.py b/web_server.py index 450c0ee6..51c4e844 100644 --- a/web_server.py +++ b/web_server.py @@ -5536,19 +5536,50 @@ def _build_search_deps(): ) +@app.route('/api/search/sources', methods=['GET']) +def search_sources(): + """Return the list of active download sources available for basic search. + + In single-source mode returns that one source. In hybrid mode returns + every source in the configured chain so the frontend can render a + source-picker chip row and let the user search specific sources. + + Response shape: ``{"mode": "hybrid"|, "sources": [{"name": str, + "display_name": str}]}`` + """ + if not download_orchestrator: + return jsonify({"mode": "soulseek", "sources": [{"name": "soulseek", "display_name": "Soulseek"}]}) + mode = download_orchestrator.mode + if mode == 'hybrid': + chain = download_orchestrator._resolve_source_chain() + sources = [ + {"name": s, "display_name": download_orchestrator.registry.display_name(s)} + for s in chain + ] + else: + sources = [{"name": mode, "display_name": download_orchestrator.registry.display_name(mode)}] + return jsonify({"mode": mode, "sources": sources}) + + @app.route('/api/search', methods=['POST']) def search_music(): - """Basic Soulseek file search.""" + """Basic download-source file search. + + Accepts an optional ``source`` body param to target a specific source + in hybrid mode (e.g. ``"soulseek"``, ``"tidal"``). When omitted, uses + the active source (single-source mode) or the first hybrid source. + """ data = request.get_json() query = data.get('query') if not query: return jsonify({"error": "No search query provided."}), 400 - logger.info(f"Web UI Search initiated for: '{query}'") + requested_source = (data.get('source') or '').strip().lower() or None + logger.info(f"Web UI Search initiated for: '{query}'" + (f" (source={requested_source})" if requested_source else "")) add_activity_item("", "Search Started", f"'{query}'", "Now") try: - results = _search_basic.run_basic_soulseek_search(query, download_orchestrator, run_async) + results = _search_basic.run_basic_search(query, download_orchestrator, run_async, source=requested_source) add_activity_item("", "Search Complete", f"'{query}' - {len(results)} results", "Now") return jsonify({"results": results}) except Exception as e: diff --git a/webui/index.html b/webui/index.html index 845a8a03..c01545e8 100644 --- a/webui/index.html +++ b/webui/index.html @@ -10,6 +10,7 @@ + {{ vite_assets('head')|safe }} @@ -2088,103 +2089,71 @@ results are cached per (query, source) pair. -->
- +
- -
- - - + + +
+ + +
+
+ + + +
+
- - - - - -
+ +
-

Ready to search • Enter artist, song, or album name

+ Enter an artist, album, or track name to search
- -
-
-

Search Results

+ + diff --git a/webui/static/basic-search-v2.css b/webui/static/basic-search-v2.css new file mode 100644 index 00000000..e028d845 --- /dev/null +++ b/webui/static/basic-search-v2.css @@ -0,0 +1,694 @@ +/* ===================================================================== + * Basic Search — visual redesign (functional behaviour untouched) + * + * Self-contained sheet appended via index.html link. Targets the new + * markup (``bs-*`` prefixed classes). Existing + * ``.search-bar-container`` / ``.filter-btn`` / ``.search-results-container`` + * rules earlier in style.css still apply elsewhere (other search UIs reuse + * them); they no longer match the new basic-search markup so this redesign + * is fully scoped. + * ===================================================================== */ + + +/* — Source chip row above the search bar. One chip per active hybrid + * source; in single-source mode the only chip is rendered with the + * ``.single`` modifier so it reads as a label rather than a control. */ +.bs-source-row { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-bottom: 12px; + align-items: center; + min-height: 32px; +} + +.bs-source-row:empty { + display: none; +} + +.bs-source-chip { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 6px 14px; + background: + linear-gradient(160deg, + rgba(255, 255, 255, 0.035) 0%, + rgba(0, 0, 0, 0.2) 100%); + border: 1px solid rgba(255, 255, 255, 0.07); + border-radius: 999px; + color: rgba(255, 255, 255, 0.55); + font-size: 11px; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; + cursor: pointer; + transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1); +} + +.bs-source-chip:hover:not(.active):not(.single):not(:disabled) { + border-color: rgba(var(--accent-rgb), 0.35); + background: + linear-gradient(160deg, + rgba(var(--accent-rgb), 0.08) 0%, + rgba(0, 0, 0, 0.2) 100%); + color: #fff; + transform: translateY(-1px); +} + +.bs-source-chip.active { + background: + linear-gradient(135deg, + color-mix(in srgb, rgb(var(--accent-rgb)) 95%, white 5%), + color-mix(in srgb, rgb(var(--accent-rgb)) 78%, black 22%)); + border-color: rgba(var(--accent-rgb), 0.55); + color: #fff; + box-shadow: + 0 4px 14px color-mix(in srgb, rgb(var(--accent-rgb)) 25%, transparent), + inset 0 1px 0 rgba(255, 255, 255, 0.18); +} + +.bs-source-chip.single { + cursor: default; + border-style: dashed; + border-color: rgba(var(--accent-rgb), 0.35); + background: transparent; + color: rgb(var(--accent-light-rgb)); +} + + +/* — Search bar: glass card matching the dashboard / auto-sync vibe. */ +.bs-search-bar { + display: flex; + gap: 12px; + align-items: center; + padding: 10px 12px; + background: + radial-gradient(ellipse at 0% 0%, + color-mix(in srgb, rgb(var(--accent-rgb)) 6%, transparent) 0%, + transparent 60%), + linear-gradient(160deg, + rgba(22, 25, 36, 0.7) 0%, + rgba(14, 16, 24, 0.85) 100%); + border: 1px solid rgba(var(--accent-rgb), 0.22); + border-radius: 16px; + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.04), + 0 6px 20px rgba(0, 0, 0, 0.25); + margin-bottom: 12px; +} + +.bs-search-input-wrap { + flex: 1; + display: flex; + align-items: center; + gap: 10px; + padding: 0 14px; + background: rgba(0, 0, 0, 0.32); + border: 1px solid rgba(255, 255, 255, 0.06); + border-radius: 12px; + height: 42px; + transition: border-color 0.2s, background 0.2s, box-shadow 0.2s; +} + +.bs-search-input-wrap:focus-within { + border-color: rgba(var(--accent-rgb), 0.45); + background: rgba(0, 0, 0, 0.4); + box-shadow: 0 0 0 3px color-mix(in srgb, rgb(var(--accent-rgb)) 12%, transparent); +} + +.bs-search-icon { + width: 16px; + height: 16px; + color: rgba(255, 255, 255, 0.4); + flex-shrink: 0; +} + +.bs-search-input-wrap:focus-within .bs-search-icon { + color: rgb(var(--accent-light-rgb)); +} + +#basic-search-section #downloads-search-input { + flex: 1; + border: none; + background: transparent; + color: #fff; + font-size: 14px; + font-weight: 500; + outline: none; + padding: 0; + height: 100%; +} + +#basic-search-section #downloads-search-input::placeholder { + color: rgba(255, 255, 255, 0.32); +} + +.bs-cancel-btn { + width: 24px; + height: 24px; + border: 1px solid rgba(255, 255, 255, 0.08); + background: rgba(255, 255, 255, 0.04); + color: rgba(255, 255, 255, 0.5); + border-radius: 50%; + cursor: pointer; + font-size: 13px; + line-height: 1; + transition: all 0.2s; + flex-shrink: 0; + padding: 0; +} + +.bs-cancel-btn:hover { + background: rgba(239, 68, 68, 0.18); + border-color: rgba(239, 68, 68, 0.45); + color: #fff; +} + +.bs-search-btn { + padding: 0 22px; + height: 42px; + border: 1px solid rgba(var(--accent-rgb), 0.55); + border-radius: 12px; + background: + linear-gradient(135deg, + color-mix(in srgb, rgb(var(--accent-rgb)) 95%, white 5%), + color-mix(in srgb, rgb(var(--accent-rgb)) 78%, black 22%)); + color: #fff; + font-size: 12px; + font-weight: 700; + letter-spacing: 0.08em; + text-transform: uppercase; + cursor: pointer; + transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1); + box-shadow: + 0 4px 14px color-mix(in srgb, rgb(var(--accent-rgb)) 28%, transparent), + inset 0 1px 0 rgba(255, 255, 255, 0.2); +} + +.bs-search-btn:hover { + transform: translateY(-1px); + box-shadow: + 0 6px 18px color-mix(in srgb, rgb(var(--accent-rgb)) 40%, transparent), + inset 0 1px 0 rgba(255, 255, 255, 0.25); +} + +.bs-search-btn:disabled { + opacity: 0.5; + cursor: not-allowed; + transform: none; +} + + +/* — Status bar: thin accent-tinted pill matching the dashboard vibe. */ +.bs-status-bar { + display: flex; + align-items: center; + gap: 10px; + padding: 8px 14px; + background: + linear-gradient(180deg, + color-mix(in srgb, rgb(var(--accent-rgb)) 4%, transparent), + transparent); + border: 1px solid rgba(255, 255, 255, 0.05); + border-radius: 10px; + margin-bottom: 12px; +} + +.bs-status-text { + color: rgba(255, 255, 255, 0.55); + font-size: 11px; + font-weight: 600; + letter-spacing: 0.02em; + margin: 0; + flex: 1; +} + + +/* — Filters: compact pill row, always visible after first search. */ +.bs-filters { + display: flex; + flex-wrap: wrap; + gap: 16px 24px; + padding: 12px 14px; + background: + linear-gradient(160deg, + rgba(255, 255, 255, 0.025) 0%, + rgba(0, 0, 0, 0.15) 100%); + border: 1px solid rgba(255, 255, 255, 0.05); + border-radius: 12px; + margin-bottom: 12px; +} + +.bs-filter-group { + display: inline-flex; + align-items: center; + gap: 6px; + flex-wrap: wrap; +} + +.bs-filter-label { + color: rgba(255, 255, 255, 0.4); + font-size: 9px; + font-weight: 800; + letter-spacing: 0.14em; + text-transform: uppercase; + margin-right: 4px; +} + +.bs-filter-pill { + padding: 4px 12px; + background: rgba(255, 255, 255, 0.03); + border: 1px solid rgba(255, 255, 255, 0.06); + border-radius: 99px; + color: rgba(255, 255, 255, 0.55); + font-size: 10px; + font-weight: 700; + letter-spacing: 0.04em; + cursor: pointer; + transition: all 0.18s ease; +} + +.bs-filter-pill:hover:not(.active) { + border-color: rgba(var(--accent-rgb), 0.35); + color: #fff; +} + +.bs-filter-pill.active { + background: + linear-gradient(135deg, + color-mix(in srgb, rgb(var(--accent-rgb)) 80%, white 20%), + rgb(var(--accent-rgb))); + border-color: rgba(var(--accent-rgb), 0.55); + color: #fff; + box-shadow: + 0 2px 8px color-mix(in srgb, rgb(var(--accent-rgb)) 25%, transparent), + inset 0 1px 0 rgba(255, 255, 255, 0.18); +} + + +/* — Results area: glass surface for cards. */ +.bs-results-wrap { + flex: 1; + min-height: 0; + overflow-y: auto; + padding: 4px 2px; + display: flex; + flex-direction: column; + gap: 10px; +} + +.bs-results-wrap .search-results-placeholder { + text-align: center; + padding: 60px 30px; + color: rgba(255, 255, 255, 0.35); + font-size: 13px; +} + + +/* — Album result card: glass + accent left-edge stripe + cover icon. + * ``!important`` is used liberally below because the original + * ``.album-result-card`` / ``.album-icon`` / etc. rules in style.css + * (~7247 onwards) are unscoped and apply heavyweight styles (24px + * padding, 56px icon, 24px border-radius, big box-shadows) that + * collide with this redesign. Scoping with ``#basic-search-section`` + * wins on specificity for some properties but the original + * ``box-shadow`` and ``padding`` rules need explicit override to + * defeat the cascade interaction with hover / pseudo states. */ +#basic-search-section .album-result-card { + background: + linear-gradient(160deg, + rgba(255, 255, 255, 0.04) 0%, + rgba(0, 0, 0, 0.2) 100%) !important; + border: 1px solid rgba(255, 255, 255, 0.06) !important; + border-radius: 14px !important; + position: relative; + transition: border-color 0.2s, transform 0.2s !important; + margin: 0 !important; + padding: 0 !important; + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.03), + 0 2px 10px rgba(0, 0, 0, 0.2) !important; + transform: none !important; +} + +#basic-search-section .album-result-card::before { + content: ''; + position: absolute; + top: 14px; + bottom: 14px; + left: 0; + width: 3px; + background: linear-gradient(180deg, + rgb(var(--accent-light-rgb)), + rgb(var(--accent-rgb))); + border-radius: 0 3px 3px 0; + box-shadow: 0 0 12px color-mix(in srgb, rgb(var(--accent-rgb)) 50%, transparent); + z-index: 2; +} + +#basic-search-section .album-result-card:hover { + border-color: rgba(var(--accent-rgb), 0.35) !important; + background: + linear-gradient(160deg, + rgba(var(--accent-rgb), 0.06) 0%, + rgba(0, 0, 0, 0.2) 100%) !important; + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.05), + 0 6px 20px rgba(0, 0, 0, 0.3) !important; + transform: translateY(-1px) !important; +} + +#basic-search-section .album-card-header { + display: flex !important; + align-items: center !important; + gap: 16px !important; + padding: 18px 22px !important; + cursor: pointer; +} + +#basic-search-section .album-expand-indicator { + color: rgba(255, 255, 255, 0.4); + font-size: 12px; + transition: transform 0.25s ease; + flex-shrink: 0; + width: 16px; + text-align: center; +} + +#basic-search-section .album-result-card.expanded .album-expand-indicator { + transform: rotate(90deg); +} + +#basic-search-section .album-icon { + width: 52px !important; + height: 52px !important; + min-width: 52px; + border-radius: 10px !important; + background: + linear-gradient(135deg, + color-mix(in srgb, rgb(var(--accent-rgb)) 35%, transparent), + rgba(0, 0, 0, 0.4)) !important; + border: 1px solid rgba(var(--accent-rgb), 0.3) !important; + display: flex !important; + align-items: center !important; + justify-content: center !important; + font-size: 26px !important; + flex-shrink: 0; + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.08) !important; + color: rgba(var(--accent-rgb), 1.0); +} + +#basic-search-section .album-info { + flex: 1 1 auto !important; + min-width: 0; + display: block; +} + +#basic-search-section .album-title { + color: rgba(255, 255, 255, 0.95) !important; + font-size: 15px !important; + font-weight: 700 !important; + letter-spacing: -0.005em; + line-height: 1.3; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + margin: 0 0 4px 0 !important; +} + +#basic-search-section .album-artist { + color: rgba(255, 255, 255, 0.65) !important; + font-size: 13px !important; + font-weight: 500; + line-height: 1.3; + margin: 0 0 6px 0 !important; +} + +#basic-search-section .album-details { + color: rgb(var(--accent-light-rgb)) !important; + font-size: 11px !important; + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; + line-height: 1.3; + margin: 0 0 3px 0 !important; +} + +#basic-search-section .album-uploader { + color: rgba(255, 255, 255, 0.4) !important; + font-size: 11px !important; + line-height: 1.3; + margin: 0 !important; +} + +#basic-search-section .album-actions { + display: flex !important; + flex-direction: column !important; + gap: 6px !important; + flex-shrink: 0; +} + + +/* — Track result card: glass row, similar height to album cards. */ +#basic-search-section .track-result-card { + display: flex !important; + align-items: center !important; + gap: 16px !important; + padding: 16px 22px !important; + background: + linear-gradient(160deg, + rgba(255, 255, 255, 0.03) 0%, + rgba(0, 0, 0, 0.18) 100%) !important; + border: 1px solid rgba(255, 255, 255, 0.05) !important; + border-radius: 14px !important; + position: relative; + transition: border-color 0.2s, transform 0.2s, background 0.2s !important; + margin: 0 !important; + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.03), + 0 2px 10px rgba(0, 0, 0, 0.2) !important; + transform: none !important; +} + +#basic-search-section .track-result-card::before { + content: ''; + position: absolute; + top: 14px; + bottom: 14px; + left: 0; + width: 2px; + background: linear-gradient(180deg, + rgba(var(--accent-rgb), 0.7), + rgba(var(--accent-rgb), 0.3)); + border-radius: 0 2px 2px 0; + z-index: 2; +} + +#basic-search-section .track-result-card:hover { + border-color: rgba(var(--accent-rgb), 0.3) !important; + background: + linear-gradient(160deg, + rgba(var(--accent-rgb), 0.06) 0%, + rgba(0, 0, 0, 0.18) 100%) !important; + transform: translateY(-1px) !important; +} + +#basic-search-section .track-icon { + width: 44px !important; + height: 44px !important; + min-width: 44px; + border-radius: 10px !important; + background: rgba(255, 255, 255, 0.04) !important; + border: 1px solid rgba(255, 255, 255, 0.08) !important; + display: flex !important; + align-items: center !important; + justify-content: center !important; + font-size: 22px !important; + flex-shrink: 0; +} + +#basic-search-section .track-info { + flex: 1 1 auto !important; + min-width: 0; +} + +#basic-search-section .track-title { + color: rgba(255, 255, 255, 0.95) !important; + font-size: 14px !important; + font-weight: 600 !important; + line-height: 1.3; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + margin: 0 0 3px 0 !important; +} + +#basic-search-section .track-artist { + color: rgba(255, 255, 255, 0.6) !important; + font-size: 12px !important; + line-height: 1.3; + margin: 0 0 5px 0 !important; +} + +#basic-search-section .track-details { + color: rgba(255, 255, 255, 0.5) !important; + font-size: 11px !important; + line-height: 1.3; + margin: 0 0 3px 0 !important; +} + +#basic-search-section .track-uploader { + color: rgba(255, 255, 255, 0.38) !important; + font-size: 11px !important; + line-height: 1.3; + margin: 0 !important; +} + +#basic-search-section .track-actions { + display: flex !important; + gap: 6px !important; + flex-shrink: 0; +} + + +/* — Action buttons inside album / track cards: pill primary + ghost. */ +#basic-search-section .album-download-btn, +#basic-search-section .album-matched-btn, +#basic-search-section .track-download-btn, +#basic-search-section .track-matched-btn, +#basic-search-section .track-stream-btn { + padding: 6px 12px; + border-radius: 99px; + font-size: 10px; + font-weight: 700; + letter-spacing: 0.04em; + cursor: pointer; + transition: all 0.18s cubic-bezier(0.4, 0, 0.2, 1); + white-space: nowrap; +} + +#basic-search-section .album-download-btn, +#basic-search-section .track-download-btn { + background: + linear-gradient(135deg, + color-mix(in srgb, rgb(var(--accent-rgb)) 92%, white 8%), + rgb(var(--accent-rgb))); + border: 1px solid rgba(var(--accent-rgb), 0.5); + color: #fff; + box-shadow: 0 2px 8px color-mix(in srgb, rgb(var(--accent-rgb)) 22%, transparent); +} + +#basic-search-section .album-download-btn:hover, +#basic-search-section .track-download-btn:hover { + transform: translateY(-1px); + box-shadow: 0 5px 14px color-mix(in srgb, rgb(var(--accent-rgb)) 35%, transparent); +} + +#basic-search-section .album-matched-btn, +#basic-search-section .track-matched-btn { + background: rgba(255, 255, 255, 0.04); + border: 1px solid rgba(var(--accent-rgb), 0.3); + color: rgb(var(--accent-light-rgb)); +} + +#basic-search-section .album-matched-btn:hover, +#basic-search-section .track-matched-btn:hover { + background: color-mix(in srgb, rgb(var(--accent-rgb)) 12%, transparent); + border-color: rgba(var(--accent-rgb), 0.5); + color: #fff; +} + +#basic-search-section .track-stream-btn { + background: rgba(255, 255, 255, 0.04); + border: 1px solid rgba(255, 255, 255, 0.1); + color: rgba(255, 255, 255, 0.65); +} + +#basic-search-section .track-stream-btn:hover { + background: rgba(255, 255, 255, 0.08); + border-color: rgba(255, 255, 255, 0.18); + color: #fff; +} + + +/* — Expanded album track list */ +#basic-search-section .album-track-list { + background: rgba(0, 0, 0, 0.18); + border-top: 1px solid rgba(255, 255, 255, 0.04); + padding: 4px 0; +} + +#basic-search-section .track-item { + display: flex; + align-items: center; + gap: 12px; + padding: 10px 18px 10px 50px; + border-bottom: 1px solid rgba(255, 255, 255, 0.03); + transition: background 0.15s; +} + +#basic-search-section .track-item:last-child { + border-bottom: none; +} + +#basic-search-section .track-item:hover { + background: rgba(255, 255, 255, 0.02); +} + +#basic-search-section .track-item-info { + flex: 1; + min-width: 0; +} + +#basic-search-section .track-item-title { + color: rgba(255, 255, 255, 0.88); + font-size: 12px; + font-weight: 500; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +#basic-search-section .track-item-details { + color: rgba(255, 255, 255, 0.4); + font-size: 10px; + margin-top: 2px; +} + +#basic-search-section .track-item-actions { + display: flex; + gap: 5px; + flex-shrink: 0; +} + +#basic-search-section .track-item-actions .track-stream-btn, +#basic-search-section .track-item-actions .track-download-btn, +#basic-search-section .track-item-actions .track-matched-btn { + padding: 4px 10px; + font-size: 9px; +} + +#basic-search-section .disc-separator { + padding: 8px 18px 6px 50px !important; + font-weight: 700 !important; + font-size: 10px !important; + color: rgb(var(--accent-light-rgb)) !important; + border-bottom: 1px solid rgba(var(--accent-rgb), 0.2) !important; + letter-spacing: 0.12em; + text-transform: uppercase; + background: none !important; + margin: 0 !important; +} + + +/* — Responsive: collapse action button row on narrow viewports. */ +@media (max-width: 900px) { + #basic-search-section .album-actions, + #basic-search-section .track-actions { + flex-direction: column; + } + .bs-filters { + gap: 10px 16px; + } +} diff --git a/webui/static/downloads.js b/webui/static/downloads.js index c44cce45..714ae934 100644 --- a/webui/static/downloads.js +++ b/webui/static/downloads.js @@ -4772,7 +4772,69 @@ function updateModalSyncProgress(playlistId, progress) { } -// Raw Soulseek file search (used by the 'Soulseek (raw files)' source picker option). +// ── Basic-search source picker ────────────────────────────────────────────── +// Tracks which download source the user has selected in the chip row. Null +// means "use the orchestrator's default" (same as pre-redesign behaviour). +let _bsActiveSource = null; + +async function initBasicSearchSources() { + const row = document.getElementById('bs-source-row'); + if (!row) return; + try { + const resp = await fetch('/api/search/sources'); + if (!resp.ok) return; + const { mode, sources } = await resp.json(); + if (!sources || !sources.length) return; + + row.innerHTML = ''; + const isSingle = mode !== 'hybrid' || sources.length < 2; + + sources.forEach((src, i) => { + const btn = document.createElement('button'); + btn.type = 'button'; + btn.className = 'bs-source-chip' + (i === 0 ? ' active' : ''); + btn.dataset.source = src.name; + btn.setAttribute('role', 'tab'); + btn.setAttribute('aria-selected', i === 0 ? 'true' : 'false'); + btn.setAttribute('title', src.display_name); + btn.innerHTML = `${_escBsHtml(src.display_name)}`; + + if (isSingle) { + // Non-interactive: single source, just a label. + btn.classList.add('single'); + btn.disabled = true; + } else { + btn.addEventListener('click', () => { + row.querySelectorAll('.bs-source-chip').forEach(c => { + c.classList.remove('active'); + c.setAttribute('aria-selected', 'false'); + }); + btn.classList.add('active'); + btn.setAttribute('aria-selected', 'true'); + _bsActiveSource = src.name; + // Re-run last search with new source if results already showing. + const query = document.getElementById('downloads-search-input')?.value?.trim(); + if (query && window.currentSearchResults?.length) { + performDownloadsSearch(); + } + }); + } + row.appendChild(btn); + }); + + // Default active source = first in chain. + _bsActiveSource = isSingle ? null : sources[0]?.name ?? null; + } catch (_err) { + // Non-fatal — search still works without the picker. + } +} + +function _escBsHtml(str) { + return String(str || '').replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); +} + + +// Raw download-source file search (basic search tab). async function performDownloadsSearch() { const query = document.getElementById('downloads-search-input').value.trim(); if (!query) { @@ -4802,10 +4864,15 @@ async function performDownloadsSearch() { displayDownloadsResults([]); // Clear previous results // --- 2. Perform the Fetch Request --- + // Source param routes the search to a specific download source in + // hybrid mode. Omitted in single-source mode (backend falls through + // to orchestrator.search() which targets the configured source). + const body = { query }; + if (_bsActiveSource) body.source = _bsActiveSource; const response = await fetch('/api/search', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ query }), + body: JSON.stringify(body), signal: searchAbortController.signal // Link fetch to the AbortController }); @@ -4825,7 +4892,8 @@ async function performDownloadsSearch() { statusText.textContent = `No results found for '${query}'`; showToast('No results found', 'error'); } else { - document.getElementById('filters-container').classList.remove('hidden'); + const filtersEl = document.getElementById('filters-container'); + if (filtersEl) filtersEl.classList.remove('hidden'); // Count albums and singles like the GUI app let totalAlbums = 0; @@ -5658,6 +5726,15 @@ let _gsController = null; else { _doInit(); setTimeout(_gsUpdateVisibility, 500); } })(); +// Init basic-search source chip row once the DOM is ready. +(function _bsInit() { + const run = () => { + if (typeof initBasicSearchSources === 'function') initBasicSearchSources(); + }; + if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', run); + else run(); +})(); + function _gsUpdateVisibility() { const bar = document.getElementById('gsearch-bar'); const aura = document.getElementById('gsearch-aura'); diff --git a/webui/static/helper.js b/webui/static/helper.js index d03f4014..2d82537a 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3415,6 +3415,7 @@ function closeHelperSearch() { const WHATS_NEW = { '2.6.5': [ { unreleased: true }, + { title: 'Basic search: visual overhaul + per-source picker in hybrid mode', desc: 'the basic search tab on the Search page was carrying its original visual treatment from before the rest of the app moved toward the glassy / accent-radial aesthetic, and it always targeted the first source in the hybrid chain with no way to pick a different one. Two things in this commit: (1) full visual redesign — glass search-bar card with accent radial wash + focus ring, pill primary search button, always-visible compact filter pill row (Type / Format / Sort), accent-tinted status pill, album result cards with accent left-edge stripe + cover icon + chevron expand + pill action buttons, track result cards as slim glass rows, multi-disc separators in album track lists, responsive button-stack on narrow viewports. Self-contained sheet at ``webui/static/basic-search-v2.css`` so revert is just dropping the link tag. (2) source picker — small chip row above the search bar lists every active source in the hybrid chain. Click a chip to target that specific source for the next search. In single-source mode the chip is rendered as a dashed-border label so the user always knows what they\'re searching. New ``GET /api/search/sources`` endpoint returns the active source list; ``/api/search`` accepts an optional ``source`` body param to target that specific source via its client. Backwards compatible — omitting ``source`` uses the orchestrator default exactly as before. Pinned with 5 new unit tests (source routing, fallback on unknown source, no-source default, alias preservation, album serialisation through the source-targeted path) on top of the existing 6 tests.', page: 'search' }, { title: 'Fix: duplicate tracks in albums with Japanese / CJK titles (#722)', desc: 'Japanese OST downloads via Apple Music + Tidal produced duplicate library entries — the same audio file landed under multiple track positions in the album. Root cause: ``MusicMatchingEngine.normalize_string`` correctly skipped unidecode for CJK text (kanji→pinyin would have been gibberish) but then ran ``re.sub(r"[^a-z0-9\\s$]", "", text)`` which stripped EVERY CJK character. Every Japanese title normalised to ``""`` and ``similarity_score`` short-circuited to 0.000 on the empty-string guard. The matcher fell back to duration+artist alone, so multiple iTunes tracks (different songs with same artist + similar duration) mapped to the same Tidal candidate. User got the same audio downloaded N times under different track positions. Fix: when CJK is detected, the alphanumeric-strip step preserves CJK Unified Ideographs, Hiragana, Katakana, Hangul, and Halfwidth/Fullwidth ranges, so CJK titles produce a comparable normalised form. Two different Japanese tracks now score appropriately low; two identical Japanese tracks now score 1.0. Latin-only normalisation is completely unchanged. 16 new unit tests pin every CJK family + every regression-prone Latin baseline. Closes #722.', page: 'downloads' }, ], '2.6.4': [ diff --git a/webui/static/style.css b/webui/static/style.css index fbadf333..78bb629d 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -7155,7 +7155,6 @@ body.helper-mode-active #dashboard-activity-feed:hover { align-items: center; gap: 18px; position: relative; - overflow: hidden; /* Neumorphic depth shadows - elevated card effect */ box-shadow: @@ -7255,7 +7254,6 @@ body.helper-mode-active #dashboard-activity-feed:hover { margin: 12px 8px; padding: 24px; position: relative; - overflow: hidden; /* Neumorphic depth shadows - elevated card effect */ box-shadow: From b14d504cc16b048e6eb98726c4a4577defc77530 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 12:00:05 -0700 Subject: [PATCH 05/52] Fix: MusicBrainz artist discography capped at 25 releases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Artist-detail discography from MusicBrainz fetched releases via the artist lookup (`/artist/?inc=release-groups`), which MusicBrainz hard-caps at 25 embedded release-groups and which ignores the `limit` param entirely. Prolific artists had ~85% of their catalogue silently dropped — Kendrick Lamar has 167 release-groups on the site but only the first 25 ever reached SoulSync. Reported by Sokhi: "a lot of albums are missing when searching vs what's showing on the site." Switch `get_artist_albums` to walk the paginated browse endpoint (`/release-group?artist=`, offset loop) — the same pattern the basic-search path already uses — fetching the full catalogue up to the caller's limit. No type filter and no studio-only filter here: the artist-detail page wants every primary/secondary type so its tabs mirror musicbrainz.org. Verified live: now returns all 167 for Kendrick. Adds 7 tests covering pagination past the cap, offset advance, short-page stop, limit cap, cross-page dedup, type->bucket mapping, and a regression pin asserting the capped inc=release-groups lookup is no longer the discography source. --- core/musicbrainz_search.py | 73 +++++++---- tests/metadata/test_musicbrainz_search.py | 147 ++++++++++++++++++++++ 2 files changed, 198 insertions(+), 22 deletions(-) diff --git a/core/musicbrainz_search.py b/core/musicbrainz_search.py index 8e7bf9bd..4929c6bd 100644 --- a/core/musicbrainz_search.py +++ b/core/musicbrainz_search.py @@ -1131,30 +1131,59 @@ class MusicBrainzSearchClient: } def get_artist_albums(self, artist_mbid: str, album_type: str = 'album,single', limit: int = 200) -> List: - """Get artist's releases for discography view.""" + """Get an artist's full discography for the artist-detail view. + + Walks the paginated browse endpoint (`/release-group?artist=`) + instead of the artist lookup's embedded release-groups. The lookup + (`/artist/?inc=release-groups`) is hard-capped at 25 release- + groups by MusicBrainz — and ignores the `limit` param entirely — so + prolific artists had ~85% of their catalogue silently dropped. That + was the bug Sokhi reported: "a lot of albums are missing vs what's + showing on the site" (e.g. Kendrick Lamar — 167 release-groups on + MB, only the first 25 ever reached SoulSync). + + Unlike the search path there is NO `type` filter and NO studio-only + filter here: the artist-detail page wants the *whole* catalogue — + albums, EPs, singles, compilations, soundtracks, live — so its tabs + mirror musicbrainz.org. `_release_group_to_album` maps each release- + group's primary/secondary types into the right bucket. + """ try: - artist = self._client.get_artist(artist_mbid, includes=['release-groups']) - if not artist or 'release-groups' not in artist: - return [] + # Lightweight lookup purely for the canonical artist name — the + # browse endpoint doesn't carry it. Non-fatal: on failure each + # Album falls back to 'Unknown Artist' via _release_group_to_album. + artist = self._client.get_artist(artist_mbid) + artist_name = (artist or {}).get('name', '') or '' - albums = [] - for rg in artist.get('release-groups', []): - primary_type = rg.get('primary-type', '') or '' - rg_type = _map_release_type(primary_type, rg.get('secondary-types', [])) - - rg_mbid = rg.get('id', '') - image_url = self._cached_art(rg_mbid, rg_mbid) - - albums.append(Album( - id=rg_mbid, - name=rg.get('title', ''), - artists=[artist.get('name', 'Unknown Artist')], - release_date=rg.get('first-release-date', '') or '', - total_tracks=0, - album_type=rg_type, - image_url=image_url, - external_urls={'musicbrainz': f'https://musicbrainz.org/release-group/{rg_mbid}'}, - )) + page_size = 100 # MusicBrainz browse hard cap per page + albums: List[Album] = [] + seen: set = set() + offset = 0 + while len(albums) < limit: + page = self._client.browse_artist_release_groups( + artist_mbid, + # No type filter — fetch every primary type. Secondary + # types (Compilation/Soundtrack/Live) ride along on their + # Album/Single/EP parent and are bucketed downstream. + # (Filtering by type=compilation is intentionally avoided: + # it's a secondary type and silently breaks MB's filter — + # see browse_artist_release_groups docs.) + release_types=None, + limit=page_size, + offset=offset, + ) + if not page: + break + for rg in page: + rg_mbid = rg.get('id', '') + if rg_mbid and rg_mbid in seen: + continue + if rg_mbid: + seen.add(rg_mbid) + albums.append(self._release_group_to_album(rg, artist_name)) + if len(page) < page_size: + break # last page reached + offset += page_size return albums[:limit] except Exception as e: logger.warning(f"MusicBrainz artist albums failed: {e}") diff --git a/tests/metadata/test_musicbrainz_search.py b/tests/metadata/test_musicbrainz_search.py index 96d2c220..d3f7defa 100644 --- a/tests/metadata/test_musicbrainz_search.py +++ b/tests/metadata/test_musicbrainz_search.py @@ -1179,3 +1179,150 @@ def test_search_tracks_with_artist_swallows_client_errors(): client._client.search_recording.side_effect = RuntimeError('network down') assert client.search_tracks_with_artist('Track', 'Artist', limit=10) == [] + + +# --------------------------------------------------------------------------- +# get_artist_albums — full-discography browse pagination +# +# Regression for Sokhi's report ("a lot of albums are missing vs the site"). +# The old impl read the artist *lookup*'s embedded release-groups +# (`/artist/?inc=release-groups`), which MusicBrainz hard-caps at 25 +# and which ignores the `limit` param — so ~85% of a prolific artist's +# catalogue (Kendrick Lamar: 167 release-groups) was silently dropped. +# The fix walks the paginated browse endpoint instead. +# --------------------------------------------------------------------------- + +def _mk_rg(rg_id, title, primary='Album', secondary=None, date='2000-01-01'): + return { + 'id': rg_id, + 'title': title, + 'primary-type': primary, + 'secondary-types': secondary or [], + 'first-release-date': date, + } + + +def test_get_artist_albums_does_not_use_capped_lookup_release_groups(): + """The capped `inc=release-groups` lookup must NOT be the source of the + discography. We still do a lightweight name lookup, but never request + the embedded (25-capped) release-groups.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_artist.return_value = {'name': 'Kendrick Lamar'} + client._client.browse_artist_release_groups.return_value = [ + _mk_rg('rg-1', 'DAMN.'), + ] + + client.get_artist_albums('mbid-kdot') + + # browse is the discography source. + assert client._client.browse_artist_release_groups.called + # The name lookup must not pull the capped embedded release-groups. + for call in client._client.get_artist.call_args_list: + assert 'release-groups' not in (call.kwargs.get('includes') or []) + assert all('release-groups' not in (a or []) for a in call.args[1:]) + + +def test_get_artist_albums_paginates_past_25_cap(): + """Walks multiple browse pages until a short page, returning the FULL + catalogue — the whole point of the fix. A single full page (100) must + trigger a follow-up fetch.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_artist.return_value = {'name': 'Prolific Artist'} + + page1 = [_mk_rg(f'rg-{i}', f'Album {i}') for i in range(100)] + page2 = [_mk_rg(f'rg-{i}', f'Album {i}') for i in range(100, 164)] + client._client.browse_artist_release_groups.side_effect = [page1, page2, []] + + albums = client.get_artist_albums('mbid-x', limit=200) + + assert len(albums) == 164 # not truncated to 25 + # Second page fetched at offset=100. + offsets = [c.kwargs.get('offset') for c in client._client.browse_artist_release_groups.call_args_list] + assert 0 in offsets and 100 in offsets + # No `type` filter — the detail page wants the whole catalogue. + for c in client._client.browse_artist_release_groups.call_args_list: + assert c.kwargs.get('release_types') is None + + +def test_get_artist_albums_stops_on_short_page(): + """A page shorter than the page size is the last page — don't fetch + a spurious extra page.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_artist.return_value = {'name': 'Small Artist'} + client._client.browse_artist_release_groups.return_value = [ + _mk_rg('rg-1', 'Only Album'), + ] + + albums = client.get_artist_albums('mbid-small', limit=200) + + assert len(albums) == 1 + client._client.browse_artist_release_groups.assert_called_once() + + +def test_get_artist_albums_respects_limit(): + """`limit` caps the returned list even when more release-groups exist.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_artist.return_value = {'name': 'Prolific Artist'} + client._client.browse_artist_release_groups.return_value = [ + _mk_rg(f'rg-{i}', f'Album {i}') for i in range(100) + ] + + albums = client.get_artist_albums('mbid-x', limit=50) + + assert len(albums) == 50 + + +def test_get_artist_albums_dedupes_release_group_ids(): + """A release-group id repeated across pages is collapsed to one card. + First page is full (100) so a second page is fetched; 'dup' appears on + both and must surface only once.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_artist.return_value = {'name': 'Artist'} + page1 = [_mk_rg('dup', 'A')] + [_mk_rg(f'rg-{i}', f'B{i}') for i in range(99)] + page2 = [_mk_rg('dup', 'A again'), _mk_rg('rg-last', 'C')] + client._client.browse_artist_release_groups.side_effect = [page1, page2, []] + + albums = client.get_artist_albums('mbid-x', limit=200) + + ids = [a.id for a in albums] + assert ids.count('dup') == 1 + assert 'rg-last' in ids + assert len(ids) == len(set(ids)) + + +def test_get_artist_albums_maps_types_into_buckets(): + """Primary/secondary types map to the album_type the discography binning + expects: EP→ep, Single→single, Album+Compilation→compilation, plain + Album→album.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_artist.return_value = {'name': 'Artist'} + client._client.browse_artist_release_groups.return_value = [ + _mk_rg('rg-lp', 'The LP', primary='Album'), + _mk_rg('rg-ep', 'The EP', primary='EP'), + _mk_rg('rg-single', 'The Single', primary='Single'), + _mk_rg('rg-comp', 'Greatest Hits', primary='Album', secondary=['Compilation']), + ] + + albums = {a.id: a for a in client.get_artist_albums('mbid-x')} + + assert albums['rg-lp'].album_type == 'album' + assert albums['rg-ep'].album_type == 'ep' + assert albums['rg-single'].album_type == 'single' + assert albums['rg-comp'].album_type == 'compilation' + + +def test_get_artist_albums_swallows_browse_errors(): + """Browse raising must not crash the discography endpoint — return [] + so the source-priority cascade falls through to the next provider.""" + client = MusicBrainzSearchClient() + client._client = MagicMock() + client._client.get_artist.return_value = {'name': 'Artist'} + client._client.browse_artist_release_groups.side_effect = RuntimeError('mb down') + + assert client.get_artist_albums('mbid-x') == [] From f7ed41867db8962756aacd43690ff59ff6289bd3 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 12:00:29 -0700 Subject: [PATCH 06/52] Fix: enhanced artist view 404s for library artists opened via source ID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opening a library artist from a non-library search result (e.g. a MusicBrainz hit) leaves the artist-detail page holding the source ID — the MBID — not the integer library PK. The standard /api/artist-detail route resolves that via find_library_artist_for_source, but the enhanced-view (`/api/library/artist//enhanced`) and quality-analysis endpoints call get_artist_full_detail directly with whatever ID the page holds. Its lookup was `WHERE id = ?` only, so it 404'd ("Artist with ID not found") and the enhanced view failed to load. When the direct PK lookup misses, fall back to matching any per-service ID column, reusing SOURCE_ID_FIELD as the single source of truth so the resolution covers every source (MusicBrainz, Spotify, Deezer, iTunes, Discogs, Hydrabase, Amazon), not just MusicBrainz. Adds 4 isolated DB-method tests: direct PK still works, resolves by MBID, resolves by Spotify ID, and unknown IDs still 404. --- database/music_database.py | 18 ++ tests/test_artist_full_detail_source_id.py | 196 +++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 tests/test_artist_full_detail_source_id.py diff --git a/database/music_database.py b/database/music_database.py index ea8eb98f..bc08e179 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -9983,6 +9983,24 @@ class MusicDatabase: # Get artist with all columns cursor.execute("SELECT * FROM artists WHERE id = ?", (artist_id,)) artist_row = cursor.fetchone() + if not artist_row: + # `artist_id` may be a *source* ID (e.g. a MusicBrainz MBID + # from a search result) rather than the integer library PK. + # The /api/artist-detail route resolves this upstream via + # find_library_artist_for_source, but the enhanced-view and + # quality-analysis endpoints call this method directly with + # whatever ID the page holds — for a library artist opened + # from a non-library search result that's the source ID, so + # the page 404'd. Resolve by matching any per-service ID + # column (single source of truth: SOURCE_ID_FIELD). + from core.artist_source_lookup import SOURCE_ID_FIELD + id_columns = list(dict.fromkeys(SOURCE_ID_FIELD.values())) + where = ' OR '.join(f"{col} = ?" for col in id_columns) + cursor.execute( + f"SELECT * FROM artists WHERE {where} LIMIT 1", + tuple(str(artist_id) for _ in id_columns), + ) + artist_row = cursor.fetchone() if not artist_row: return {'success': False, 'error': f'Artist with ID {artist_id} not found'} diff --git a/tests/test_artist_full_detail_source_id.py b/tests/test_artist_full_detail_source_id.py new file mode 100644 index 00000000..7f05e619 --- /dev/null +++ b/tests/test_artist_full_detail_source_id.py @@ -0,0 +1,196 @@ +"""Regression tests for ``MusicDatabase.get_artist_full_detail`` source-ID +resolution. + +Bug (reported by Boulder, 2026-05-28): opening a library artist from a +*non-library* search result (e.g. a MusicBrainz hit) leaves the artist-detail +page holding the source ID — the MBID — not the integer library PK. The +standard /api/artist-detail route resolves that via +``find_library_artist_for_source``, but the **enhanced-view** endpoint +(``/api/library/artist//enhanced``) and the quality-analysis endpoint call +``get_artist_full_detail`` directly with whatever ID the page holds. With only +a ``WHERE id = ?`` lookup, that 404'd ("Artist with ID not found") and +the enhanced view failed to load. + +Fix: when the direct PK lookup misses, resolve against any per-service ID +column (``SOURCE_ID_FIELD``). + +These are isolated DB-method tests — no Flask, no route layer — so the SQL +fallback itself is exercised. +""" + +import sqlite3 +import sys +import types + +import pytest + + +# ── stubs (same shape used elsewhere in the test suite) ─────────────────── +if "spotipy" not in sys.modules: + spotipy = types.ModuleType("spotipy") + spotipy.Spotify = object + oauth2 = types.ModuleType("spotipy.oauth2") + oauth2.SpotifyOAuth = object + oauth2.SpotifyClientCredentials = object + spotipy.oauth2 = oauth2 + sys.modules["spotipy"] = spotipy + sys.modules["spotipy.oauth2"] = oauth2 + +if "config.settings" not in sys.modules: + config_pkg = types.ModuleType("config") + settings_mod = types.ModuleType("config.settings") + + class _DummyConfigManager: + def get(self, key, default=None): + return default + + def get_active_media_server(self): + return "primary" + + settings_mod.config_manager = _DummyConfigManager() + config_pkg.settings = settings_mod + sys.modules["config"] = config_pkg + sys.modules["config.settings"] = settings_mod + + +from database.music_database import MusicDatabase # noqa: E402 + + +class _InMemoryDB(MusicDatabase): + """MusicDatabase backed by an in-memory sqlite that survives across + ``_get_connection()`` calls.""" + + def __init__(self): + self._conn = sqlite3.connect(":memory:") + self._conn.row_factory = sqlite3.Row + + def _get_connection(self): + return _NonClosingConn(self._conn) + + +class _NonClosingConn: + def __init__(self, real): + self._real = real + + def cursor(self): + return self._real.cursor() + + def commit(self): + return self._real.commit() + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + +def _seed_schema(db): + cur = db._conn.cursor() + # Only the columns get_artist_full_detail touches (it uses SELECT *, so + # the per-service ID columns must exist for the resolution fallback). + cur.execute(""" + CREATE TABLE artists ( + id INTEGER PRIMARY KEY, + name TEXT, + server_source TEXT, + genres TEXT, + musicbrainz_id TEXT, + spotify_artist_id TEXT, + deezer_id TEXT, + itunes_artist_id TEXT, + discogs_id TEXT, + soul_id TEXT, + amazon_id TEXT + ) + """) + cur.execute(""" + CREATE TABLE albums ( + id TEXT PRIMARY KEY, + artist_id INTEGER, + title TEXT, + year INTEGER, + genres TEXT, + record_type TEXT, + track_count INTEGER + ) + """) + cur.execute(""" + CREATE TABLE tracks ( + id TEXT PRIMARY KEY, + album_id TEXT, + title TEXT, + track_number INTEGER + ) + """) + db._conn.commit() + + +def _seed_kendrick(db, **id_columns): + """Insert a Kendrick Lamar library artist (PK 187926) with one album, + setting whichever per-service ID columns the test needs.""" + cols = ['id', 'name', 'server_source'] + list(id_columns) + vals = [187926, 'Kendrick Lamar', 'primary'] + list(id_columns.values()) + placeholders = ','.join('?' * len(cols)) + cur = db._conn.cursor() + cur.execute(f"INSERT INTO artists ({','.join(cols)}) VALUES ({placeholders})", vals) + cur.execute( + "INSERT INTO albums (id, artist_id, title, year, record_type) VALUES (?, ?, ?, ?, ?)", + ('alb-1', 187926, 'DAMN.', 2017, 'album'), + ) + db._conn.commit() + + +@pytest.fixture +def db(): + d = _InMemoryDB() + _seed_schema(d) + return d + + +def test_direct_pk_lookup_still_works(db): + """The primary path — integer library PK — must be unaffected by the + new fallback.""" + _seed_kendrick(db, musicbrainz_id='381086ea-mbid') + + result = db.get_artist_full_detail(187926) + + assert result['success'] is True + assert result['artist']['name'] == 'Kendrick Lamar' + assert [a['title'] for a in result['albums']] == ['DAMN.'] + + +def test_resolves_by_musicbrainz_id(db): + """The exact bug: page holds the MBID, not the PK. Must resolve and + return the library artist + albums instead of 404ing.""" + _seed_kendrick(db, musicbrainz_id='381086ea-mbid') + + result = db.get_artist_full_detail('381086ea-mbid') + + assert result['success'] is True + assert result['artist']['name'] == 'Kendrick Lamar' + assert [a['title'] for a in result['albums']] == ['DAMN.'] + + +def test_resolves_by_spotify_id(db): + """Resolution isn't MusicBrainz-specific — any per-service ID column + works (proves SOURCE_ID_FIELD reuse, not a hardcoded mbid check).""" + _seed_kendrick(db, spotify_artist_id='sp-kdot') + + result = db.get_artist_full_detail('sp-kdot') + + assert result['success'] is True + assert result['artist']['name'] == 'Kendrick Lamar' + + +def test_unknown_id_returns_not_found(db): + """An ID that matches neither the PK nor any source column still 404s.""" + _seed_kendrick(db, musicbrainz_id='381086ea-mbid') + + result = db.get_artist_full_detail('totally-unknown-id') + + assert result['success'] is False + assert 'not found' in result['error'] From ff974c0b5c12806bd0daba9954da04ff43a26734 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 12:35:02 -0700 Subject: [PATCH 07/52] Standardize artist-detail hero action buttons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four artist-hero buttons (Artist Radio, Watchlist, Download Discography, Enhance Quality) had drifted apart visually — different sizes, weights, radii, hover treatments, and ad-hoc colors. Unify them on the Download Discography look (the nicest of the set): accent-style gradient fill, matching border, light-tinted text, compact 7x16 / 12px / 700 sizing, and a translateY(-1px) + colored glow on hover. To keep them distinguishable, each carries its own hue within that shared recipe: - Artist Radio -> violet (#8b5cf6) - Watchlist -> theme accent (keeps its amber "watching" state) - Download Discography -> theme accent + shimmer (the primary action) - Enhance Quality -> cyan (#4fc3f7, its original signature color) Also: - Drop the shimmer from the three secondary buttons — four simultaneous shimmers were distracting; it now marks only the primary action. - Remove the `margin: 12px 0 4px` on `.discog-download-wrap` (now `margin: 0; display: inline-flex`) that pushed the discography button ~4px below its siblings in the centered flex row. - Include Artist Radio in the mobile button sizing override (was missing). --- webui/static/mobile.css | 2 + webui/static/style.css | 173 +++++++++++++++------------------------- 2 files changed, 67 insertions(+), 108 deletions(-) diff --git a/webui/static/mobile.css b/webui/static/mobile.css index b920708f..d25b8f0f 100644 --- a/webui/static/mobile.css +++ b/webui/static/mobile.css @@ -2923,6 +2923,7 @@ ====================================== */ @media (max-width: 768px) { + .library-artist-radio-btn, .library-artist-watchlist-btn, .library-artist-enhance-btn { padding: 8px 14px; @@ -2930,6 +2931,7 @@ gap: 6px; } + .library-artist-radio-btn .radio-icon, .library-artist-watchlist-btn .watchlist-icon, .library-artist-enhance-btn .enhance-icon { font-size: 12px; diff --git a/webui/static/style.css b/webui/static/style.css index 78bb629d..7c2aa9d2 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -23890,46 +23890,36 @@ body.helper-mode-active #dashboard-activity-feed:hover { /* Library Artist Watchlist Button */ +/* Artist-hero action buttons share the Download Discography look: + accent gradient fill, accent border, accent-light text, compact sizing, + subtle shimmer, lift + glow on hover. The watchlist .watching state keeps + its amber "already watching" signal; everything else is uniform. */ .library-artist-watchlist-btn { + position: relative; display: flex; align-items: center; + justify-content: center; gap: 8px; - padding: 10px 20px; - font-size: 13px; - font-weight: 600; + padding: 7px 16px; + font-size: 12px; + font-weight: 700; letter-spacing: 0.02em; - color: rgba(255, 255, 255, 0.85); - background: linear-gradient(135deg, rgba(255, 255, 255, 0.06) 0%, rgba(255, 255, 255, 0.02) 100%); - border: 1px solid rgba(255, 255, 255, 0.08); - border-radius: 12px; + color: rgb(var(--accent-light-rgb)); + background: linear-gradient(135deg, rgba(var(--accent-rgb), 0.15) 0%, rgba(var(--accent-rgb), 0.05) 100%); + border: 1px solid rgba(var(--accent-rgb), 0.25); + border-radius: 8px; cursor: pointer; - transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1); + transition: all 0.25s ease; outline: none; - font-family: 'SF Pro Text', -apple-system, BlinkMacSystemFont, sans-serif; - position: relative; + font-family: inherit; overflow: hidden; - backdrop-filter: blur(8px); -} - -.library-artist-watchlist-btn::before { - content: ''; - position: absolute; - inset: 0; - border-radius: 12px; - background: linear-gradient(135deg, rgba(var(--accent-rgb), 0.2) 0%, rgba(var(--accent-rgb), 0.05) 100%); - opacity: 0; - transition: opacity 0.3s ease; -} - -.library-artist-watchlist-btn:hover:not(:disabled)::before { - opacity: 1; } .library-artist-watchlist-btn:hover:not(:disabled) { - color: #ffffff; - border-color: rgba(var(--accent-rgb), 0.35); - transform: translateY(-2px); - box-shadow: 0 4px 20px rgba(var(--accent-rgb), 0.2), 0 0 0 1px rgba(var(--accent-rgb), 0.1); + background: linear-gradient(135deg, rgba(var(--accent-rgb), 0.25) 0%, rgba(var(--accent-rgb), 0.1) 100%); + border-color: rgba(var(--accent-rgb), 0.4); + transform: translateY(-1px); + box-shadow: 0 4px 16px rgba(var(--accent-rgb), 0.2); } .library-artist-watchlist-btn:active:not(:disabled) { @@ -23938,14 +23928,14 @@ body.helper-mode-active #dashboard-activity-feed:hover { } .library-artist-watchlist-btn .watchlist-icon { - font-size: 14px; - transition: transform 0.3s cubic-bezier(0.4, 0, 0.2, 1); + font-size: 13px; + transition: transform 0.25s ease; position: relative; z-index: 1; } .library-artist-watchlist-btn:hover:not(:disabled) .watchlist-icon { - transform: scale(1.15); + transform: scale(1.12); } .library-artist-watchlist-btn .watchlist-text { @@ -23954,20 +23944,16 @@ body.helper-mode-active #dashboard-activity-feed:hover { } .library-artist-watchlist-btn.watching { - background: linear-gradient(135deg, rgba(255, 193, 7, 0.15) 0%, rgba(255, 193, 7, 0.05) 100%); color: #ffc107; - border-color: rgba(255, 193, 7, 0.25); - box-shadow: 0 0 12px rgba(255, 193, 7, 0.08); -} - -.library-artist-watchlist-btn.watching::before { - background: linear-gradient(135deg, rgba(255, 193, 7, 0.25) 0%, rgba(255, 193, 7, 0.08) 100%); + background: linear-gradient(135deg, rgba(255, 193, 7, 0.15) 0%, rgba(255, 193, 7, 0.05) 100%); + border-color: rgba(255, 193, 7, 0.3); } .library-artist-watchlist-btn.watching:hover:not(:disabled) { - color: #ffffff; - border-color: rgba(255, 193, 7, 0.5); - box-shadow: 0 4px 20px rgba(255, 193, 7, 0.25), 0 0 0 1px rgba(255, 193, 7, 0.15); + background: linear-gradient(135deg, rgba(255, 193, 7, 0.25) 0%, rgba(255, 193, 7, 0.1) 100%); + border-color: rgba(255, 193, 7, 0.45); + transform: translateY(-1px); + box-shadow: 0 4px 16px rgba(255, 193, 7, 0.2); } .library-artist-watchlist-btn:disabled { @@ -23980,45 +23966,31 @@ body.helper-mode-active #dashboard-activity-feed:hover { /* ─── Enhance Quality Button ─── */ .library-artist-enhance-btn { + position: relative; display: flex; align-items: center; + justify-content: center; gap: 8px; - padding: 10px 20px; - font-size: 13px; - font-weight: 600; + padding: 7px 16px; + font-size: 12px; + font-weight: 700; letter-spacing: 0.02em; - color: rgba(79, 195, 247, 0.9); - background: linear-gradient(135deg, rgba(79, 195, 247, 0.08) 0%, rgba(79, 195, 247, 0.02) 100%); - border: 1px solid rgba(79, 195, 247, 0.15); - border-radius: 12px; + color: rgb(129, 212, 250); + background: linear-gradient(135deg, rgba(79, 195, 247, 0.15) 0%, rgba(79, 195, 247, 0.05) 100%); + border: 1px solid rgba(79, 195, 247, 0.3); + border-radius: 8px; cursor: pointer; - transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1); + transition: all 0.25s ease; outline: none; - font-family: 'SF Pro Text', -apple-system, BlinkMacSystemFont, sans-serif; - position: relative; + font-family: inherit; overflow: hidden; - backdrop-filter: blur(8px); -} - -.library-artist-enhance-btn::before { - content: ''; - position: absolute; - inset: 0; - border-radius: 12px; - background: linear-gradient(135deg, rgba(79, 195, 247, 0.22) 0%, rgba(100, 220, 255, 0.08) 100%); - opacity: 0; - transition: opacity 0.3s ease; -} - -.library-artist-enhance-btn:hover::before { - opacity: 1; } .library-artist-enhance-btn:hover { - color: #ffffff; - border-color: rgba(79, 195, 247, 0.4); - transform: translateY(-2px); - box-shadow: 0 4px 20px rgba(79, 195, 247, 0.2), 0 0 0 1px rgba(79, 195, 247, 0.1); + background: linear-gradient(135deg, rgba(79, 195, 247, 0.25) 0%, rgba(79, 195, 247, 0.12) 100%); + border-color: rgba(79, 195, 247, 0.5); + transform: translateY(-1px); + box-shadow: 0 4px 16px rgba(79, 195, 247, 0.25); } .library-artist-enhance-btn:active { @@ -24027,14 +23999,14 @@ body.helper-mode-active #dashboard-activity-feed:hover { } .library-artist-enhance-btn .enhance-icon { - font-size: 14px; - transition: transform 0.3s cubic-bezier(0.4, 0, 0.2, 1); + font-size: 13px; + transition: transform 0.25s ease; position: relative; z-index: 1; } .library-artist-enhance-btn:hover .enhance-icon { - transform: scale(1.2) rotate(8deg); + transform: scale(1.12); } .library-artist-enhance-btn .enhance-text { @@ -24082,45 +24054,31 @@ body.helper-mode-active #dashboard-activity-feed:hover { /* ─── Artist Radio Button ─── */ .library-artist-radio-btn { + position: relative; display: flex; align-items: center; + justify-content: center; gap: 8px; - padding: 10px 20px; - font-size: 13px; - font-weight: 600; + padding: 7px 16px; + font-size: 12px; + font-weight: 700; letter-spacing: 0.02em; - color: rgba(255, 255, 255, 0.85); - background: linear-gradient(135deg, rgba(255, 255, 255, 0.06) 0%, rgba(255, 255, 255, 0.02) 100%); - border: 1px solid rgba(255, 255, 255, 0.08); - border-radius: 12px; + color: rgb(196, 181, 253); + background: linear-gradient(135deg, rgba(139, 92, 246, 0.15) 0%, rgba(139, 92, 246, 0.05) 100%); + border: 1px solid rgba(139, 92, 246, 0.3); + border-radius: 8px; cursor: pointer; - transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1); + transition: all 0.25s ease; outline: none; - font-family: 'SF Pro Text', -apple-system, BlinkMacSystemFont, sans-serif; - position: relative; + font-family: inherit; overflow: hidden; - backdrop-filter: blur(8px); -} - -.library-artist-radio-btn::before { - content: ''; - position: absolute; - inset: 0; - border-radius: 12px; - background: linear-gradient(135deg, rgba(var(--accent-rgb), 0.2) 0%, rgba(var(--accent-rgb), 0.05) 100%); - opacity: 0; - transition: opacity 0.3s ease; -} - -.library-artist-radio-btn:hover::before { - opacity: 1; } .library-artist-radio-btn:hover { - color: #ffffff; - border-color: rgba(var(--accent-rgb), 0.35); - transform: translateY(-2px); - box-shadow: 0 4px 20px rgba(var(--accent-rgb), 0.2), 0 0 0 1px rgba(var(--accent-rgb), 0.1); + background: linear-gradient(135deg, rgba(139, 92, 246, 0.25) 0%, rgba(139, 92, 246, 0.12) 100%); + border-color: rgba(139, 92, 246, 0.5); + transform: translateY(-1px); + box-shadow: 0 4px 16px rgba(139, 92, 246, 0.25); } .library-artist-radio-btn:active { @@ -24129,14 +24087,14 @@ body.helper-mode-active #dashboard-activity-feed:hover { } .library-artist-radio-btn .radio-icon { - font-size: 14px; - transition: transform 0.3s cubic-bezier(0.4, 0, 0.2, 1); + font-size: 13px; + transition: transform 0.25s ease; position: relative; z-index: 1; } .library-artist-radio-btn:hover .radio-icon { - transform: scale(1.15); + transform: scale(1.12); } .library-artist-radio-btn .radio-text { @@ -46910,7 +46868,7 @@ textarea.enhanced-meta-field-input { DOWNLOAD DISCOGRAPHY — Button + Modal ═══════════════════════════════════════════════════════════════════════════ */ -.discog-download-wrap { margin: 12px 0 4px; } +.discog-download-wrap { margin: 0; display: inline-flex; } .discog-download-btn { position: relative; @@ -46947,7 +46905,6 @@ textarea.enhanced-meta-field-input { padding: 7px 16px; font-size: 12px; border-radius: 8px; - margin-top: 8px; } .discog-btn-compact .discog-btn-icon { font-size: 12px; } From c9ad4f496fb54478b7a549b58260b52ba026c3df Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 13:21:53 -0700 Subject: [PATCH 08/52] Embed highest-resolution album art across all art paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User report: embedded album art came out ~600x600 while the cover.jpg in the folder was high-res. The cover.jpg path upgraded the source CDN URL to its highest resolution, but the tag-embed path fetched the raw URL — so iTunes art embedded at its 600x600 default, Spotify at 640, Deezer at 1000. The "Write Tags to File" retag path had the same gap (Deezer-only upgrade), and MusicBrainz art was worse still: every Cover Art Archive URL is built as the /front-250 thumbnail, so MB-sourced downloads embedded 250x250. Factor the resolution upgrade + fetch into two shared helpers in core/metadata/artwork.py and route every art path through them: _upgrade_art_url(url) — bump to the source's highest resolution: - Spotify (i.scdn.co) -> original master (~2000px+) - iTunes (mzstatic.com) -> 3000x3000 - Deezer (dzcdn) -> 1900x1900 - Cover Art Archive -> /front original (was /front-250) _fetch_art_bytes(url) — upgrade, fetch, and fall back once to the original size if the CDN refuses the larger one (non-regressive). Now consistent across: embed-into-tags (post-process), folder cover.jpg (post-process), and the enhanced-library "Write Tags to File" retag flow. The YouTube path already upgraded via Album.from_spotify_album, unchanged. De-duplicates the per-source upgrade code that was copied across sites and drops the now-unused urllib import from tag_writer. Not covered (follow-up): Last.fm / Amazon / Tidal / Qobuz have no explicit upgrade yet — some already serve full-res, others may hand over a capped size that passes through unchanged. Tests: new tests/metadata/test_artwork_resolution.py pins every upgrade (Spotify 300/640->master, iTunes 100/600->3000, Deezer->1900, CAA thumbnail->original, unrecognized/empty unchanged) and the fetch fallback. Updated the two tag_writer fallback tests to patch the network at its new home in artwork. --- core/metadata/artwork.py | 131 +++++++++------ core/tag_writer.py | 49 +----- tests/metadata/test_artwork_resolution.py | 158 ++++++++++++++++++ .../metadata/test_deezer_cover_url_upgrade.py | 15 +- 4 files changed, 254 insertions(+), 99 deletions(-) create mode 100644 tests/metadata/test_artwork_resolution.py diff --git a/core/metadata/artwork.py b/core/metadata/artwork.py index 694933d9..f1ef58d8 100644 --- a/core/metadata/artwork.py +++ b/core/metadata/artwork.py @@ -256,6 +256,79 @@ def _browser_safe_image_url(url: str) -> str: return url +def _upgrade_art_url(art_url: str) -> str: + """Rewrite a source CDN art URL to the highest resolution that source + serves, so embedded tag art is as sharp as the cover.jpg in the folder. + + - Spotify (i.scdn.co): request the original uploaded master (~2000px+). + - iTunes (mzstatic.com): bump the size segment to 3000x3000. + - Deezer (dzcdn): rewrite to 1900x1900 (CDN serves larger than the API's + 1000px cover_xl). + + Unrecognized URLs are returned unchanged. Both the embed and cover.jpg + paths call this so the two never diverge in quality again. + """ + if not art_url: + return art_url + if "i.scdn.co" in art_url: + try: + from core.spotify_client import _upgrade_spotify_image_url + + return _upgrade_spotify_image_url(art_url) + except Exception as e: + logger.debug("upgrade spotify image url failed: %s", e) + elif "mzstatic.com" in art_url: + return re.sub(r"\d+x\d+bb", "3000x3000bb", art_url) + elif "dzcdn" in art_url: + try: + from core.deezer_client import _upgrade_deezer_cover_url + + return _upgrade_deezer_cover_url(art_url) + except Exception as e: + logger.debug("upgrade deezer image url failed: %s", e) + elif "coverartarchive.org" in art_url: + # MusicBrainz art comes in as Cover Art Archive thumbnails + # (/front-250, also -500/-1200 — see musicbrainz_search._cover_art_url). + # The bare /front is the original full-resolution upload, which always + # exists when any front art does, so rewrite the size segment back to + # it. Critical for MB-as-source users: without this, embedded art is + # the 250px search thumbnail. `_fetch_art_bytes` falls back to the + # sized URL if /front is ever refused. + return re.sub(r"/front-\d+", "/front", art_url) + return art_url + + +def _fetch_art_bytes(art_url: str): + """Fetch artwork bytes at the highest resolution the source serves. + + Upgrades the URL via `_upgrade_art_url`, then fetches. If the upgraded + (larger) size is refused by the CDN, retry once with the original URL so + we never regress vs. the un-upgraded behavior. Empirically the upgrade + works for every album tested; the fallback just defends the edge case. + + Returns `(image_data, mime_type)` or `(None, None)` on failure. + """ + if not art_url: + return None, None + upgraded = _upgrade_art_url(art_url) + try: + with urllib.request.urlopen(upgraded, timeout=10) as response: + return response.read(), (response.info().get_content_type() or "image/jpeg") + except Exception as fetch_err: + if upgraded != art_url: + logger.info( + "Upgraded art URL refused (%s); retrying original size", fetch_err + ) + try: + with urllib.request.urlopen(art_url, timeout=10) as response: + return response.read(), (response.info().get_content_type() or "image/jpeg") + except Exception as retry_err: + logger.error("Art fetch failed after fallback: %s", retry_err) + return None, None + logger.error("Art fetch failed: %s", fetch_err) + return None, None + + def embed_album_art_metadata(audio_file, metadata: dict): cfg = get_config_manager() symbols = get_mutagen_symbols() @@ -284,9 +357,7 @@ def embed_album_art_metadata(audio_file, metadata: dict): if not art_url: logger.warning("No album art URL available for embedding.") return - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - mime_type = response.info().get_content_type() or "image/jpeg" + image_data, mime_type = _fetch_art_bytes(art_url) if not image_data: logger.error("Failed to download album art data.") @@ -365,59 +436,13 @@ def download_cover_art(album_info: dict, target_dir: str, context: dict = None): art_url = images[0].get("url", "") if art_url: logger.info("Using cover art URL from album context") - if art_url and "i.scdn.co" in art_url: - try: - from core.spotify_client import _upgrade_spotify_image_url - - art_url = _upgrade_spotify_image_url(art_url) - except Exception as e: - logger.debug("upgrade spotify image url failed: %s", e) - elif art_url and "mzstatic.com" in art_url: - art_url = re.sub(r"\d+x\d+bb", "3000x3000bb", art_url) - elif art_url and "dzcdn" in art_url: - # Deezer's API returns cover_xl URLs at 1000×1000 but - # the underlying CDN serves up to 1900×1900 by rewriting - # the size segment in the URL path. Without this upgrade - # users embedding cover art via Deezer get visibly - # blurry covers in their library / phone player (Discord - # report from Tim, 2026-05). Same shape as the iTunes - # mzstatic upgrade above + Spotify scdn upgrade. - try: - from core.deezer_client import _upgrade_deezer_cover_url - - art_url = _upgrade_deezer_cover_url(art_url) - except Exception as e: - logger.debug("upgrade deezer image url failed: %s", e) if not art_url: logger.warning("No cover art URL available for download.") return - # Fetch with one fallback level: if we upgraded a Deezer - # URL above and the CDN happens to refuse the larger size - # for this specific album, retry with the original URL so - # we never regress vs. pre-upgrade behavior. Empirically - # 1900 works for every album tested but defending against - # the edge case keeps the fix strictly non-regressive. - original_url = album_info.get("album_image_url") - if context and not original_url: - album_ctx = get_import_context_album(context) - original_url = album_ctx.get("image_url") or original_url - try: - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - except Exception as fetch_err: - if ( - "dzcdn" in art_url - and original_url - and original_url != art_url - ): - logger.info( - "Deezer CDN refused upgraded cover URL (%s); " - "retrying with original size", fetch_err, - ) - with urllib.request.urlopen(original_url, timeout=10) as response: - image_data = response.read() - else: - raise + # Upgrade to the source's highest resolution (Spotify master / + # iTunes 3000 / Deezer 1900) with a one-level fallback — shared + # with the tag-embed path so cover.jpg and embedded art match. + image_data, _ = _fetch_art_bytes(art_url) if not image_data: return diff --git a/core/tag_writer.py b/core/tag_writer.py index 5b4d5752..e6edbc12 100644 --- a/core/tag_writer.py +++ b/core/tag_writer.py @@ -6,7 +6,6 @@ Reuses the same Mutagen patterns as _enhance_file_metadata in web_server.py. import os import logging -import urllib.request from typing import Dict, Any, Optional, List, Tuple from mutagen import File as MutagenFile @@ -205,48 +204,18 @@ def download_cover_art(cover_url: str) -> Optional[Tuple[bytes, str]]: Download cover art once. Returns (image_data, mime_type) or None on failure. Call this once per album, then pass the result to write_tags_to_file for each track. - For Deezer CDN URLs, upgrades the size segment to 1900×1900 (CDN - max). Mirrors the same upgrade in - ``core.metadata.artwork.download_cover_art`` so the - enhanced-library-view "Write Tags to File" feature embeds the same - high-resolution cover the auto post-process flow does. Falls back - to the original URL if the CDN refuses the upgraded size for a - specific album — keeps the fix strictly non-regressive vs. the - pre-upgrade behaviour. + Delegates to ``core.metadata.artwork._fetch_art_bytes`` so the enhanced- + library-view "Write Tags to File" feature embeds the same highest- + resolution cover the auto post-process flow does — Spotify master + (~2000px), iTunes 3000×3000, and Deezer 1900×1900 — with the same + one-level fallback to the original size if the CDN refuses the upgrade. """ if not cover_url: return None - original_url = cover_url - if 'dzcdn' in cover_url: - try: - from core.deezer_client import _upgrade_deezer_cover_url - cover_url = _upgrade_deezer_cover_url(cover_url) - except Exception as e: - logger.debug("upgrade deezer image url failed: %s", e) - try: - with urllib.request.urlopen(cover_url, timeout=15) as response: - image_data = response.read() - mime_type = response.info().get_content_type() or 'image/jpeg' - if image_data: - return (image_data, mime_type) - except Exception as e: - # Deezer CDN refused upgraded size for this album — retry with - # original URL so we never get less than pre-upgrade behaviour. - if 'dzcdn' in cover_url and cover_url != original_url: - logger.info( - "Deezer CDN refused upgraded cover URL (%s); retrying with original size", e, - ) - try: - with urllib.request.urlopen(original_url, timeout=15) as response: - image_data = response.read() - mime_type = response.info().get_content_type() or 'image/jpeg' - if image_data: - return (image_data, mime_type) - except Exception as fallback_err: - logger.error(f"Error downloading cover art (fallback): {fallback_err}") - return None - logger.error(f"Error downloading cover art from {cover_url}: {e}") - return None + from core.metadata.artwork import _fetch_art_bytes + + image_data, mime_type = _fetch_art_bytes(cover_url) + return (image_data, mime_type) if image_data else None def write_tags_to_file(file_path: str, db_data: Dict[str, Any], diff --git a/tests/metadata/test_artwork_resolution.py b/tests/metadata/test_artwork_resolution.py new file mode 100644 index 00000000..3b9c976b --- /dev/null +++ b/tests/metadata/test_artwork_resolution.py @@ -0,0 +1,158 @@ +"""Pin the shared artwork resolution-upgrade + fetch helpers. + +Bug (Discord, user report): embedded album art came out ~600×600 while the +cover.jpg in the folder was high-res. Cause: only the cover.jpg path +upgraded the source CDN URL to its highest resolution (Spotify master / +iTunes 3000 / Deezer 1900); the tag-embed path and the "Write Tags to File" +retag path fetched the raw URL — Spotify 640, iTunes 600, Deezer 1000. + +Fix: one shared ``_upgrade_art_url`` + ``_fetch_art_bytes`` in +``core.metadata.artwork`` that every art path now calls, so embedded art is +always the same highest resolution as the folder cover. +""" + +from __future__ import annotations + +import pytest + +from core.metadata.artwork import _upgrade_art_url, _fetch_art_bytes + + +# --------------------------------------------------------------------------- +# _upgrade_art_url — per-source resolution bump +# --------------------------------------------------------------------------- + +class TestUpgradeArtUrl: + def test_spotify_album_art_upgraded_to_master(self): + # Spotify encodes size as the hex segment after 'ab67616d0000'. + # 1e02 = 300px, b273 = 640px; 82c1 = the original uploaded master. + url = 'https://i.scdn.co/image/ab67616d00001e02deadbeef' + assert _upgrade_art_url(url) == 'https://i.scdn.co/image/ab67616d000082c1deadbeef' + + def test_spotify_640_also_upgraded(self): + url = 'https://i.scdn.co/image/ab67616d0000b273cafef00d' + assert _upgrade_art_url(url) == 'https://i.scdn.co/image/ab67616d000082c1cafef00d' + + def test_itunes_600_upgraded_to_3000(self): + url = 'https://is1-ssl.mzstatic.com/image/thumb/abc/600x600bb.jpg' + assert _upgrade_art_url(url) == 'https://is1-ssl.mzstatic.com/image/thumb/abc/3000x3000bb.jpg' + + def test_itunes_100_upgraded_to_3000(self): + url = 'https://is1-ssl.mzstatic.com/image/thumb/abc/100x100bb.jpg' + assert '3000x3000bb' in _upgrade_art_url(url) + + def test_deezer_upgraded_to_1900(self): + url = 'https://cdn-images.dzcdn.net/images/cover/abc/1000x1000-000000-80-0-0.jpg' + assert '1900x1900' in _upgrade_art_url(url) + + def test_caa_thumbnail_upgraded_to_original(self): + # MusicBrainz art arrives as /front-250 (and -500/-1200) thumbnails; + # /front is the original full-resolution upload. + url = 'https://coverartarchive.org/release/abc-123/front-250' + assert _upgrade_art_url(url) == 'https://coverartarchive.org/release/abc-123/front' + + def test_caa_500_and_release_group_scope_upgraded(self): + assert _upgrade_art_url('https://coverartarchive.org/release/x/front-500') \ + == 'https://coverartarchive.org/release/x/front' + assert _upgrade_art_url('https://coverartarchive.org/release-group/y/front-1200') \ + == 'https://coverartarchive.org/release-group/y/front' + + def test_caa_already_original_unchanged(self): + url = 'https://coverartarchive.org/release/abc/front' + assert _upgrade_art_url(url) == url + + @pytest.mark.parametrize('url', [ + 'https://lastfm.freetls.fastly.net/i/u/770x0/x.jpg', + 'https://example.com/random.jpg', + ]) + def test_unrecognized_url_unchanged(self, url): + assert _upgrade_art_url(url) == url + + def test_empty_and_none_unchanged(self): + assert _upgrade_art_url('') == '' + assert _upgrade_art_url(None) is None + + +# --------------------------------------------------------------------------- +# _fetch_art_bytes — upgrade, fetch, fall back to original on CDN refusal +# --------------------------------------------------------------------------- + +class _FakeResponse: + def __init__(self, data=b'art-bytes', ctype='image/jpeg'): + self._data = data + self._ctype = ctype + + def read(self): + return self._data + + def info(self): + ctype = self._ctype + + class _Info: + def get_content_type(_self): + return ctype + + return _Info() + + def __enter__(self): + return self + + def __exit__(self, *a): + pass + + +class TestFetchArtBytes: + def test_fetches_upgraded_url(self, monkeypatch): + """The upgraded (high-res) URL is the one actually fetched.""" + calls = [] + + def fake_urlopen(url, timeout=None): + calls.append(url) + return _FakeResponse(b'big-cover', 'image/png') + + monkeypatch.setattr('core.metadata.artwork.urllib.request.urlopen', fake_urlopen) + + data, mime = _fetch_art_bytes('https://i.scdn.co/image/ab67616d0000b273x') + + assert data == b'big-cover' + assert mime == 'image/png' + # Fetched the master-res URL, not the 640 original. + assert calls == ['https://i.scdn.co/image/ab67616d000082c1x'] + + def test_falls_back_to_original_when_upgrade_refused(self, monkeypatch): + upgraded = 'https://is1-ssl.mzstatic.com/image/thumb/a/3000x3000bb.jpg' + original = 'https://is1-ssl.mzstatic.com/image/thumb/a/600x600bb.jpg' + calls = [] + + def fake_urlopen(url, timeout=None): + calls.append(url) + if url == upgraded: + raise Exception('403 Forbidden') + return _FakeResponse(b'orig-cover') + + monkeypatch.setattr('core.metadata.artwork.urllib.request.urlopen', fake_urlopen) + + data, mime = _fetch_art_bytes(original) + + assert data == b'orig-cover' + assert calls == [upgraded, original] # tried big first, then fell back + + def test_no_fallback_when_url_not_upgraded(self, monkeypatch): + """If the upgrade is a no-op (unrecognized URL), a single failed + fetch returns (None, None) — no pointless retry of the same URL.""" + calls = [] + + def fake_urlopen(url, timeout=None): + calls.append(url) + raise Exception('network down') + + monkeypatch.setattr('core.metadata.artwork.urllib.request.urlopen', fake_urlopen) + + data, mime = _fetch_art_bytes('https://example.com/cover.jpg') + + assert (data, mime) == (None, None) + assert calls == ['https://example.com/cover.jpg'] + + def test_empty_url_returns_none(self): + assert _fetch_art_bytes('') == (None, None) + assert _fetch_art_bytes(None) == (None, None) diff --git a/tests/metadata/test_deezer_cover_url_upgrade.py b/tests/metadata/test_deezer_cover_url_upgrade.py index caba3dea..16c80f1b 100644 --- a/tests/metadata/test_deezer_cover_url_upgrade.py +++ b/tests/metadata/test_deezer_cover_url_upgrade.py @@ -153,7 +153,9 @@ class TestDownloadFallbackOnCdnRefusal: def test_tag_writer_retries_with_original_on_failure(self, monkeypatch): """tag_writer.download_cover_art must fall back to the - original URL when the upgraded URL fails.""" + original URL when the upgraded URL fails. The fetch now lives in + the shared ``core.metadata.artwork._fetch_art_bytes`` helper that + tag_writer delegates to, so the network is patched there.""" from core import tag_writer original_url = 'https://cdn-images.dzcdn.net/images/cover/abc/1000x1000-000000-80-0-0.jpg' @@ -176,7 +178,7 @@ class TestDownloadFallbackOnCdnRefusal: raise Exception("403 Forbidden") return _FakeResponse() - monkeypatch.setattr('core.tag_writer.urllib.request.urlopen', fake_urlopen) + monkeypatch.setattr('core.metadata.artwork.urllib.request.urlopen', fake_urlopen) result = tag_writer.download_cover_art(original_url) @@ -185,8 +187,9 @@ class TestDownloadFallbackOnCdnRefusal: assert call_log == [upgraded_url, original_url] def test_tag_writer_no_fallback_for_non_dzcdn_url(self, monkeypatch): - """Non-Deezer URLs go through unchanged — no upgrade, no - fallback. Fast path preserved.""" + """Non-Deezer URLs with no recognizable size segment go through + unchanged — no upgrade, so a single fetch attempt with no + fallback.""" from core import tag_writer spotify_url = 'https://i.scdn.co/image/abc' @@ -196,10 +199,10 @@ class TestDownloadFallbackOnCdnRefusal: call_log.append(url) raise Exception("network error") - monkeypatch.setattr('core.tag_writer.urllib.request.urlopen', fake_urlopen) + monkeypatch.setattr('core.metadata.artwork.urllib.request.urlopen', fake_urlopen) result = tag_writer.download_cover_art(spotify_url) assert result is None - # Single attempt — no Deezer fallback path triggered + # Single attempt — upgrade is a no-op for this URL, so no fallback assert call_log == [spotify_url] From abdea631a70e79027f4b9b9d87423d0fea75cedc Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 14:37:03 -0700 Subject: [PATCH 09/52] HiFi/MB cover art: use CAA 1200px thumbnail, not the flaky /front original MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the album-art resolution fix. That change upgraded MusicBrainz Cover Art Archive thumbnails (/front-250) to the bare /front original — but /front redirects to archive.org, which is unreliable: probing release-group covers showed intermittent HTTP 500s (same URL 500s one second, serves the next) and multi-MB originals (2.9 MB seen). The result was the user-reported flakiness: cover art that "sometimes works, sometimes shows nothing", and a huge image embedded into every track when it did work. The sized thumbnails (/front-250, -500, -1200) are served by CAA's own CDN, not the archive.org redirect — which is why /front-250 (240p) was always reliable. Upgrade to /front-1200 instead: 1200x1200 is a massive jump from 240p, reliably CDN-served, and a sane ~40 KB instead of multi-MB. Applied in all three CAA spots for consistency: the _upgrade_art_url helper (embed + cover.jpg paths) and both prefer_caa ("CCA") blocks, which fetched the bare /front directly with no fallback — so CCA-on users hit the same flakiness. _fetch_art_bytes still falls back to the original /front-250 if /front-1200 is ever refused. Tests updated to assert the 1200px target, idempotency, and that the bare /front original is intentionally left untouched. --- core/metadata/artwork.py | 23 +++++++++++++---------- tests/metadata/test_artwork_resolution.py | 23 +++++++++++++++-------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/core/metadata/artwork.py b/core/metadata/artwork.py index f1ef58d8..0da28c58 100644 --- a/core/metadata/artwork.py +++ b/core/metadata/artwork.py @@ -287,14 +287,15 @@ def _upgrade_art_url(art_url: str) -> str: except Exception as e: logger.debug("upgrade deezer image url failed: %s", e) elif "coverartarchive.org" in art_url: - # MusicBrainz art comes in as Cover Art Archive thumbnails - # (/front-250, also -500/-1200 — see musicbrainz_search._cover_art_url). - # The bare /front is the original full-resolution upload, which always - # exists when any front art does, so rewrite the size segment back to - # it. Critical for MB-as-source users: without this, embedded art is - # the 250px search thumbnail. `_fetch_art_bytes` falls back to the - # sized URL if /front is ever refused. - return re.sub(r"/front-\d+", "/front", art_url) + # MusicBrainz art arrives as Cover Art Archive thumbnails + # (/front-250 — see musicbrainz_search._cover_art_url). Upgrade to the + # 1200px thumbnail: a huge jump from 240p yet still served by CAA's own + # CDN. Deliberately NOT the bare /front original — that redirects to + # archive.org, which is flaky (intermittent 500s/timeouts) and can be + # multi-MB, nasty to embed in every track. 1200 is the sweet spot of + # quality + reliability. `_fetch_art_bytes` falls back to the original + # sized URL if /front-1200 is ever refused. + return re.sub(r"/front-\d+", "/front-1200", art_url) return art_url @@ -342,7 +343,8 @@ def embed_album_art_metadata(audio_file, metadata: dict): release_mbid = metadata.get("musicbrainz_release_id") if release_mbid and cfg.get("metadata_enhancement.prefer_caa_art", False): try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + # 1200px CDN thumbnail, not the flaky bare /front original. + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front-1200" req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) with urllib.request.urlopen(req, timeout=10) as response: image_data = response.read() @@ -412,7 +414,8 @@ def download_cover_art(album_info: dict, target_dir: str, context: dict = None): image_data = None if release_mbid and prefer_caa: try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + # 1200px CDN thumbnail, not the flaky bare /front original. + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front-1200" req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) with urllib.request.urlopen(req, timeout=10) as response: image_data = response.read() diff --git a/tests/metadata/test_artwork_resolution.py b/tests/metadata/test_artwork_resolution.py index 3b9c976b..e07c56d0 100644 --- a/tests/metadata/test_artwork_resolution.py +++ b/tests/metadata/test_artwork_resolution.py @@ -45,19 +45,26 @@ class TestUpgradeArtUrl: url = 'https://cdn-images.dzcdn.net/images/cover/abc/1000x1000-000000-80-0-0.jpg' assert '1900x1900' in _upgrade_art_url(url) - def test_caa_thumbnail_upgraded_to_original(self): - # MusicBrainz art arrives as /front-250 (and -500/-1200) thumbnails; - # /front is the original full-resolution upload. + def test_caa_thumbnail_upgraded_to_1200(self): + # MusicBrainz art arrives as the /front-250 thumbnail; upgrade to the + # 1200px CDN thumbnail (NOT the flaky bare /front original). url = 'https://coverartarchive.org/release/abc-123/front-250' - assert _upgrade_art_url(url) == 'https://coverartarchive.org/release/abc-123/front' + assert _upgrade_art_url(url) == 'https://coverartarchive.org/release/abc-123/front-1200' - def test_caa_500_and_release_group_scope_upgraded(self): + def test_caa_500_scope_and_idempotent(self): assert _upgrade_art_url('https://coverartarchive.org/release/x/front-500') \ - == 'https://coverartarchive.org/release/x/front' + == 'https://coverartarchive.org/release/x/front-1200' + # release-group scope works the same. + assert _upgrade_art_url('https://coverartarchive.org/release-group/y/front-250') \ + == 'https://coverartarchive.org/release-group/y/front-1200' + # Idempotent — an already-1200 URL stays put. assert _upgrade_art_url('https://coverartarchive.org/release-group/y/front-1200') \ - == 'https://coverartarchive.org/release-group/y/front' + == 'https://coverartarchive.org/release-group/y/front-1200' - def test_caa_already_original_unchanged(self): + def test_caa_bare_front_left_alone(self): + # The bare /front original is intentionally NOT what we want; the + # sized-thumbnail regex doesn't touch it (and it never reaches the + # helper in practice — _cover_art_url always emits /front-250). url = 'https://coverartarchive.org/release/abc/front' assert _upgrade_art_url(url) == url From 628395eda5593db184ddd19d1821ba9df0b548bd Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 15:57:23 -0700 Subject: [PATCH 10/52] Discovery lift (1/N): convert_*_results_to_spotify_tracks -> shared helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First cluster of the per-source playlist-discovery deduplication. The convert__results_to_spotify_tracks functions (Tidal, Deezer, Qobuz, Spotify-Public, YouTube, ListenBrainz) plus the already-generic _convert_link_results_to_spotify_tracks were byte-identical apart from the source label used in their log line. Lift the shared body into core/discovery/endpoints.py as convert_results_to_spotify_tracks(results, source_label); the 7 web_server functions become 1-line delegations (names/signatures unchanged, so all callers and behavior are identical — 1:1). Beatport is intentionally NOT folded in: its converter coerces artist objects to strings and emits a different track shape (source field, album dict), so it keeps its own implementation. Tests: tests/discovery/test_discovery_endpoints.py (12) pin both input shapes (manual spotify_data / auto spotify_track+found), optional track/disc numbers, falsy-0 omission, field defaults, skip-on-neither, order preservation, if/elif precedence, empty input. web_server.py: -209 lines. Full discovery suite: 151 passed. --- core/discovery/endpoints.py | 76 +++++++ tests/discovery/test_discovery_endpoints.py | 127 +++++++++++ web_server.py | 227 +------------------- 3 files changed, 212 insertions(+), 218 deletions(-) create mode 100644 core/discovery/endpoints.py create mode 100644 tests/discovery/test_discovery_endpoints.py diff --git a/core/discovery/endpoints.py b/core/discovery/endpoints.py new file mode 100644 index 00000000..b8c4e411 --- /dev/null +++ b/core/discovery/endpoints.py @@ -0,0 +1,76 @@ +"""Generic, source-agnostic helpers for the playlist-discovery route layer. + +The discovery/sync endpoints in ``web_server.py`` were copy-pasted once per +source (Tidal, Deezer, Qobuz, Spotify-public, iTunes-link, YouTube, +ListenBrainz, Beatport). The per-source copies differ only by a source label +string and which ``_discovery_states`` global they read. This module +lifts the source-agnostic pieces into importable, unit-testable helpers so the +route functions become thin wrappers — exactly preserving behavior (1:1). + +Each helper is lifted verbatim from its web_server.py counterpart; any +per-source quirk that genuinely differs (e.g. Beatport's distinct result +shape) is intentionally NOT routed through here and stays in its own function. +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +from utils.logging_config import get_logger + +logger = get_logger("discovery.endpoints") + + +def convert_results_to_spotify_tracks( + discovery_results: List[Dict[str, Any]], + source_label: str, +) -> List[Dict[str, Any]]: + """Convert a source's discovery results into the Spotify-track dicts the + sync pipeline expects. + + Lifted verbatim from the per-source ``convert__results_to_spotify_tracks`` + functions (and the already-generic ``_convert_link_results_to_spotify_tracks``), + which were byte-identical apart from the ``source_label`` used in the log + line. Two input shapes are supported, matching the originals exactly: + + - ``spotify_data`` (manual-fix shape): copied through, preserving optional + ``track_number`` / ``disc_number``. + - ``spotify_track`` + ``status_class == 'found'`` (auto-discovery shape): + rebuilt from the flat ``spotify_*`` fields. + + Any result matching neither shape is skipped, identical to the originals. + + NOTE: Beatport deliberately does NOT use this — its converter coerces + artist objects to strings and emits a different track shape (``source`` + field, album dict), so it keeps its own implementation. + """ + spotify_tracks: List[Dict[str, Any]] = [] + + for result in discovery_results: + # Support both data formats: spotify_data (manual fixes) and individual + # fields (automatic discovery). + if result.get('spotify_data'): + spotify_data = result['spotify_data'] + track = { + 'id': spotify_data['id'], + 'name': spotify_data['name'], + 'artists': spotify_data['artists'], + 'album': spotify_data['album'], + 'duration_ms': spotify_data.get('duration_ms', 0), + } + if spotify_data.get('track_number'): + track['track_number'] = spotify_data['track_number'] + if spotify_data.get('disc_number'): + track['disc_number'] = spotify_data['disc_number'] + spotify_tracks.append(track) + elif result.get('spotify_track') and result.get('status_class') == 'found': + spotify_tracks.append({ + 'id': result.get('spotify_id', 'unknown'), + 'name': result.get('spotify_track', 'Unknown Track'), + 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], + 'album': result.get('spotify_album', 'Unknown Album'), + 'duration_ms': 0, + }) + + logger.info(f"Converted {len(spotify_tracks)} {source_label} matches to Spotify tracks for sync") + return spotify_tracks diff --git a/tests/discovery/test_discovery_endpoints.py b/tests/discovery/test_discovery_endpoints.py new file mode 100644 index 00000000..0b78c1b4 --- /dev/null +++ b/tests/discovery/test_discovery_endpoints.py @@ -0,0 +1,127 @@ +"""Tests for the lifted, source-agnostic discovery route helpers in +``core.discovery.endpoints``. + +These pin the exact behavior the per-source ``convert__results_to_spotify_tracks`` +functions had in web_server.py, so the lift is provably 1:1. Each input shape +the originals handled is exercised here. +""" + +from __future__ import annotations + +from core.discovery.endpoints import convert_results_to_spotify_tracks + + +# --------------------------------------------------------------------------- +# spotify_data (manual-fix) shape +# --------------------------------------------------------------------------- + +def test_spotify_data_shape_basic(): + results = [{ + 'spotify_data': { + 'id': 'sp1', 'name': 'Song', 'artists': ['A'], 'album': 'Alb', + 'duration_ms': 1234, + } + }] + assert convert_results_to_spotify_tracks(results, 'Tidal') == [{ + 'id': 'sp1', 'name': 'Song', 'artists': ['A'], 'album': 'Alb', + 'duration_ms': 1234, + }] + + +def test_spotify_data_duration_defaults_to_zero(): + results = [{'spotify_data': {'id': 'x', 'name': 'n', 'artists': [], 'album': 'a'}}] + out = convert_results_to_spotify_tracks(results, 'Deezer') + assert out[0]['duration_ms'] == 0 + + +def test_spotify_data_includes_track_and_disc_number_when_present(): + results = [{'spotify_data': { + 'id': 'x', 'name': 'n', 'artists': [], 'album': 'a', + 'track_number': 5, 'disc_number': 2, + }}] + out = convert_results_to_spotify_tracks(results, 'Qobuz') + assert out[0]['track_number'] == 5 + assert out[0]['disc_number'] == 2 + + +def test_spotify_data_omits_track_disc_when_absent_or_falsy(): + # track_number/disc_number of 0 are falsy -> omitted, matching original. + results = [{'spotify_data': { + 'id': 'x', 'name': 'n', 'artists': [], 'album': 'a', + 'track_number': 0, 'disc_number': 0, + }}] + out = convert_results_to_spotify_tracks(results, 'YouTube') + assert 'track_number' not in out[0] + assert 'disc_number' not in out[0] + + +# --------------------------------------------------------------------------- +# spotify_track + status_class == 'found' (auto-discovery) shape +# --------------------------------------------------------------------------- + +def test_auto_discovery_shape_full(): + results = [{ + 'spotify_track': 'Track', 'status_class': 'found', + 'spotify_id': 'id9', 'spotify_artist': 'Artist', 'spotify_album': 'Album', + }] + assert convert_results_to_spotify_tracks(results, 'ListenBrainz') == [{ + 'id': 'id9', 'name': 'Track', 'artists': ['Artist'], 'album': 'Album', + 'duration_ms': 0, + }] + + +def test_auto_discovery_defaults_when_fields_missing(): + results = [{'spotify_track': 'T', 'status_class': 'found'}] + out = convert_results_to_spotify_tracks(results, 'Spotify Public') + assert out == [{ + 'id': 'unknown', 'name': 'T', 'artists': ['Unknown Artist'], + 'album': 'Unknown Album', 'duration_ms': 0, + }] + + +def test_auto_discovery_empty_artist_yields_unknown_artist(): + results = [{ + 'spotify_track': 'T', 'status_class': 'found', 'spotify_artist': '', + }] + out = convert_results_to_spotify_tracks(results, 'Tidal') + assert out[0]['artists'] == ['Unknown Artist'] + + +# --------------------------------------------------------------------------- +# skip / mixed / empty +# --------------------------------------------------------------------------- + +def test_auto_discovery_requires_found_status(): + # spotify_track present but status_class != 'found' -> skipped. + results = [{'spotify_track': 'T', 'status_class': 'not_found'}] + assert convert_results_to_spotify_tracks(results, 'Tidal') == [] + + +def test_result_matching_neither_shape_is_skipped(): + results = [{'irrelevant': True}, {'spotify_track': 'T'}] # 2nd has no status_class + assert convert_results_to_spotify_tracks(results, 'Tidal') == [] + + +def test_mixed_results_preserve_order(): + results = [ + {'spotify_data': {'id': '1', 'name': 'a', 'artists': [], 'album': ''}}, + {'irrelevant': True}, + {'spotify_track': 'b', 'status_class': 'found', 'spotify_id': '2'}, + ] + out = convert_results_to_spotify_tracks(results, 'Tidal') + assert [t['id'] for t in out] == ['1', '2'] + + +def test_empty_input(): + assert convert_results_to_spotify_tracks([], 'Tidal') == [] + + +def test_spotify_data_takes_precedence_over_auto_fields(): + # A result carrying both shapes uses spotify_data (the if-branch wins), + # matching the original if/elif ordering. + results = [{ + 'spotify_data': {'id': 'D', 'name': 'd', 'artists': [], 'album': ''}, + 'spotify_track': 'IGNORED', 'status_class': 'found', 'spotify_id': 'A', + }] + out = convert_results_to_spotify_tracks(results, 'Tidal') + assert out[0]['id'] == 'D' diff --git a/web_server.py b/web_server.py index 51c4e844..27742136 100644 --- a/web_server.py +++ b/web_server.py @@ -21073,6 +21073,8 @@ from core.discovery.scoring import ( # Tidal discovery worker logic lives in core/discovery/tidal.py. from core.discovery import tidal as _discovery_tidal +# Source-agnostic discovery route helpers (lifted from the per-source copies). +from core.discovery.endpoints import convert_results_to_spotify_tracks def _build_tidal_discovery_deps(): @@ -21103,39 +21105,7 @@ def _run_tidal_discovery_worker(playlist_id): def convert_tidal_results_to_spotify_tracks(discovery_results): """Convert Tidal discovery results to Spotify tracks format for sync""" - spotify_tracks = [] - - for result in discovery_results: - # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery) - if result.get('spotify_data'): - spotify_data = result['spotify_data'] - - # Create track object matching the expected format - track = { - 'id': spotify_data['id'], - 'name': spotify_data['name'], - 'artists': spotify_data['artists'], - 'album': spotify_data['album'], - 'duration_ms': spotify_data.get('duration_ms', 0) - } - if spotify_data.get('track_number'): - track['track_number'] = spotify_data['track_number'] - if spotify_data.get('disc_number'): - track['disc_number'] = spotify_data['disc_number'] - spotify_tracks.append(track) - elif result.get('spotify_track') and result.get('status_class') == 'found': - # Build from individual fields (automatic discovery format) - track = { - 'id': result.get('spotify_id', 'unknown'), - 'name': result.get('spotify_track', 'Unknown Track'), - 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], - 'album': result.get('spotify_album', 'Unknown Album'), - 'duration_ms': 0 - } - spotify_tracks.append(track) - - logger.info(f"Converted {len(spotify_tracks)} Tidal matches to Spotify tracks for sync") - return spotify_tracks + return convert_results_to_spotify_tracks(discovery_results, "Tidal") # =================================================================== @@ -21817,37 +21787,7 @@ def _run_deezer_discovery_worker(playlist_id): def convert_deezer_results_to_spotify_tracks(discovery_results): """Convert Deezer discovery results to Spotify tracks format for sync""" - spotify_tracks = [] - - for result in discovery_results: - # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery) - if result.get('spotify_data'): - spotify_data = result['spotify_data'] - - track = { - 'id': spotify_data['id'], - 'name': spotify_data['name'], - 'artists': spotify_data['artists'], - 'album': spotify_data['album'], - 'duration_ms': spotify_data.get('duration_ms', 0) - } - if spotify_data.get('track_number'): - track['track_number'] = spotify_data['track_number'] - if spotify_data.get('disc_number'): - track['disc_number'] = spotify_data['disc_number'] - spotify_tracks.append(track) - elif result.get('spotify_track') and result.get('status_class') == 'found': - track = { - 'id': result.get('spotify_id', 'unknown'), - 'name': result.get('spotify_track', 'Unknown Track'), - 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], - 'album': result.get('spotify_album', 'Unknown Album'), - 'duration_ms': 0 - } - spotify_tracks.append(track) - - logger.info(f"Converted {len(spotify_tracks)} Deezer matches to Spotify tracks for sync") - return spotify_tracks + return convert_results_to_spotify_tracks(discovery_results, "Deezer") # =================================================================== @@ -22478,36 +22418,7 @@ def _run_qobuz_discovery_worker(playlist_id): def convert_qobuz_results_to_spotify_tracks(discovery_results): """Convert Qobuz discovery results to Spotify tracks format for sync.""" - spotify_tracks = [] - - for result in discovery_results: - if result.get('spotify_data'): - spotify_data = result['spotify_data'] - - track = { - 'id': spotify_data['id'], - 'name': spotify_data['name'], - 'artists': spotify_data['artists'], - 'album': spotify_data['album'], - 'duration_ms': spotify_data.get('duration_ms', 0) - } - if spotify_data.get('track_number'): - track['track_number'] = spotify_data['track_number'] - if spotify_data.get('disc_number'): - track['disc_number'] = spotify_data['disc_number'] - spotify_tracks.append(track) - elif result.get('spotify_track') and result.get('status_class') == 'found': - track = { - 'id': result.get('spotify_id', 'unknown'), - 'name': result.get('spotify_track', 'Unknown Track'), - 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], - 'album': result.get('spotify_album', 'Unknown Album'), - 'duration_ms': 0 - } - spotify_tracks.append(track) - - logger.info(f"Converted {len(spotify_tracks)} Qobuz matches to Spotify tracks for sync") - return spotify_tracks + return convert_results_to_spotify_tracks(discovery_results, "Qobuz") # =================================================================== @@ -22967,32 +22878,7 @@ def _build_itunes_link_state(response_data): def _convert_link_results_to_spotify_tracks(discovery_results, label): - spotify_tracks = [] - for result in discovery_results: - if result.get('spotify_data'): - spotify_data = result['spotify_data'] - track = { - 'id': spotify_data['id'], - 'name': spotify_data['name'], - 'artists': spotify_data['artists'], - 'album': spotify_data['album'], - 'duration_ms': spotify_data.get('duration_ms', 0) - } - if spotify_data.get('track_number'): - track['track_number'] = spotify_data['track_number'] - if spotify_data.get('disc_number'): - track['disc_number'] = spotify_data['disc_number'] - spotify_tracks.append(track) - elif result.get('spotify_track') and result.get('status_class') == 'found': - spotify_tracks.append({ - 'id': result.get('spotify_id', 'unknown'), - 'name': result.get('spotify_track', 'Unknown Track'), - 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], - 'album': result.get('spotify_album', 'Unknown Album'), - 'duration_ms': 0 - }) - logger.info(f"Converted {len(spotify_tracks)} {label} matches to Spotify tracks for sync") - return spotify_tracks + return convert_results_to_spotify_tracks(discovery_results, label) @app.route('/api/spotify/parse-public', methods=['POST']) def parse_spotify_public_endpoint(): @@ -23442,38 +23328,7 @@ def _run_spotify_public_discovery_worker(url_hash): def convert_spotify_public_results_to_spotify_tracks(discovery_results): """Convert Spotify Public discovery results to Spotify tracks format for sync""" - spotify_tracks = [] - - for result in discovery_results: - # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery) - if result.get('spotify_data'): - spotify_data = result['spotify_data'] - - track = { - 'id': spotify_data['id'], - 'name': spotify_data['name'], - 'artists': spotify_data['artists'], - 'album': spotify_data['album'], - 'duration_ms': spotify_data.get('duration_ms', 0) - } - # Preserve track_number/disc_number from discovery enrichment - if spotify_data.get('track_number'): - track['track_number'] = spotify_data['track_number'] - if spotify_data.get('disc_number'): - track['disc_number'] = spotify_data['disc_number'] - spotify_tracks.append(track) - elif result.get('spotify_track') and result.get('status_class') == 'found': - track = { - 'id': result.get('spotify_id', 'unknown'), - 'name': result.get('spotify_track', 'Unknown Track'), - 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], - 'album': result.get('spotify_album', 'Unknown Album'), - 'duration_ms': 0 - } - spotify_tracks.append(track) - - logger.info(f"Converted {len(spotify_tracks)} Spotify Public matches to Spotify tracks for sync") - return spotify_tracks + return convert_results_to_spotify_tracks(discovery_results, "Spotify Public") # =================================================================== @@ -24889,39 +24744,7 @@ def update_youtube_playlist_phase(url_hash): def convert_youtube_results_to_spotify_tracks(discovery_results): """Convert YouTube discovery results to Spotify tracks format for sync""" - spotify_tracks = [] - - for result in discovery_results: - # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery) - if result.get('spotify_data'): - spotify_data = result['spotify_data'] - - # Create track object matching the expected format - track = { - 'id': spotify_data['id'], - 'name': spotify_data['name'], - 'artists': spotify_data['artists'], - 'album': spotify_data['album'], - 'duration_ms': spotify_data.get('duration_ms', 0) - } - if spotify_data.get('track_number'): - track['track_number'] = spotify_data['track_number'] - if spotify_data.get('disc_number'): - track['disc_number'] = spotify_data['disc_number'] - spotify_tracks.append(track) - elif result.get('spotify_track') and result.get('status_class') == 'found': - # Build from individual fields (automatic discovery format) - track = { - 'id': result.get('spotify_id', 'unknown'), - 'name': result.get('spotify_track', 'Unknown Track'), - 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], - 'album': result.get('spotify_album', 'Unknown Album'), - 'duration_ms': 0 - } - spotify_tracks.append(track) - - logger.info(f"Converted {len(spotify_tracks)} YouTube matches to Spotify tracks for sync") - return spotify_tracks + return convert_results_to_spotify_tracks(discovery_results, "YouTube") # Add these new endpoints to the end of web_server.py @@ -30702,39 +30525,7 @@ def update_listenbrainz_discovery_match(): def convert_listenbrainz_results_to_spotify_tracks(discovery_results): """Convert ListenBrainz discovery results to Spotify tracks format for sync""" - spotify_tracks = [] - - for result in discovery_results: - # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery) - if result.get('spotify_data'): - spotify_data = result['spotify_data'] - - # Create track object matching the expected format - track = { - 'id': spotify_data['id'], - 'name': spotify_data['name'], - 'artists': spotify_data['artists'], - 'album': spotify_data['album'], - 'duration_ms': spotify_data.get('duration_ms', 0) - } - if spotify_data.get('track_number'): - track['track_number'] = spotify_data['track_number'] - if spotify_data.get('disc_number'): - track['disc_number'] = spotify_data['disc_number'] - spotify_tracks.append(track) - elif result.get('spotify_track') and result.get('status_class') == 'found': - # Build from individual fields (automatic discovery format) - track = { - 'id': result.get('spotify_id', 'unknown'), - 'name': result.get('spotify_track', 'Unknown Track'), - 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], - 'album': result.get('spotify_album', 'Unknown Album'), - 'duration_ms': 0 - } - spotify_tracks.append(track) - - logger.info(f"Converted {len(spotify_tracks)} ListenBrainz matches to Spotify tracks for sync") - return spotify_tracks + return convert_results_to_spotify_tracks(discovery_results, "ListenBrainz") @app.route('/api/wing-it/sync', methods=['POST']) def wing_it_sync(): From 2d76a7c06176d6afee5e79bf548401a954b1a5eb Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 16:12:51 -0700 Subject: [PATCH 11/52] Discovery lift (2/N): cancel_*_sync + delete_*_playlist -> shared helpers Second cluster. Two more sets of byte-identical per-source bodies: cancel__sync (Tidal, Deezer, Qobuz, Spotify-Public, iTunes-Link, YouTube, ListenBrainz) -> core.discovery.endpoints.cancel_sync(states, key, *, label, not_found_message, sync_lock, sync_states, active_sync_workers). Returns (payload, status_code); a thin web_server glue (_cancel_source_sync) wires the sync-infra globals + jsonify. Caller passes the resolved key (ListenBrainz transforms via _lb_state_key) and the exact 404 string (iTunes-Link uses "iTunes Link not found"). delete__playlist (Tidal, Deezer, Qobuz, Spotify-Public) -> delete_playlist_state(states, key, *, label, not_found_message), wired via _delete_source_playlist. Intentionally left with their own bodies (genuinely divergent, not 1:1): - Beatport cancel (cancels a stored sync_future, no message, warning log). - iTunes-Link / YouTube / ListenBrainz / Beatport deletes (different success messages, info-log wording, playlist-name extraction, /remove route, chart key). Tests: +11 in tests/discovery/test_discovery_endpoints.py covering cancel (404, active-worker cancel + state revert, worker-absent, no-sync-in-progress, label in message, exception->500) and delete (404, future cancel + removal, no/falsy future, exception->500 leaves state). Full discovery suite: 162 passed. web_server.py: -216 lines. --- core/discovery/endpoints.py | 85 +++++- tests/discovery/test_discovery_endpoints.py | 136 ++++++++- web_server.py | 288 +++----------------- 3 files changed, 255 insertions(+), 254 deletions(-) diff --git a/core/discovery/endpoints.py b/core/discovery/endpoints.py index b8c4e411..61d8cc0b 100644 --- a/core/discovery/endpoints.py +++ b/core/discovery/endpoints.py @@ -14,7 +14,8 @@ shape) is intentionally NOT routed through here and stays in its own function. from __future__ import annotations -from typing import Any, Dict, List +import time +from typing import Any, Dict, List, Tuple from utils.logging_config import get_logger @@ -74,3 +75,85 @@ def convert_results_to_spotify_tracks( logger.info(f"Converted {len(spotify_tracks)} {source_label} matches to Spotify tracks for sync") return spotify_tracks + + +def cancel_sync( + states: Dict[str, Any], + key: str, + *, + label: str, + not_found_message: str, + sync_lock: Any, + sync_states: Dict[str, Any], + active_sync_workers: Dict[str, Any], +) -> Tuple[Dict[str, Any], int]: + """Cancel an in-progress sync for one discovery playlist. + + 1:1 lift of the byte-identical ``cancel__sync`` bodies (Tidal, + Deezer, Qobuz, Spotify-Public, iTunes-Link, YouTube, ListenBrainz). The + caller passes the already-resolved state key (ListenBrainz transforms it + via ``_lb_state_key`` first), the source ``label``, the exact 404 message + (iTunes-Link uses "iTunes Link not found", not "... playlist not found"), + and the shared sync infrastructure (so this stays free of web_server + globals / Flask). + + Returns ``(payload_dict, status_code)``; the caller wraps in ``jsonify``. + + Beatport is NOT routed here — it cancels a stored ``sync_future`` and + returns a different payload. + """ + try: + if key not in states: + return {"error": not_found_message}, 404 + + state = states[key] + state['last_accessed'] = time.time() + sync_playlist_id = state.get('sync_playlist_id') + + if sync_playlist_id: + with sync_lock: + sync_states[sync_playlist_id] = {"status": "cancelled"} + if sync_playlist_id in active_sync_workers: + del active_sync_workers[sync_playlist_id] + + state['phase'] = 'discovered' + state['sync_playlist_id'] = None + state['sync_progress'] = {} + + return {"success": True, "message": f"{label} sync cancelled"}, 200 + except Exception as e: + logger.error(f"Error cancelling {label} sync: {e}") + return {"error": str(e)}, 500 + + +def delete_playlist_state( + states: Dict[str, Any], + key: str, + *, + label: str, + not_found_message: str, +) -> Tuple[Dict[str, Any], int]: + """Delete a discovery playlist's state entry, cancelling any active + discovery first. + + 1:1 lift of the byte-identical ``delete__playlist`` bodies + (Tidal, Deezer, Qobuz, Spotify-Public). Returns ``(payload, status_code)``. + + The iTunes-Link / YouTube / ListenBrainz / Beatport deletes intentionally + keep their own bodies — they differ in success message, info-log wording, + name extraction, and/or key transform. + """ + try: + if key not in states: + return {"error": not_found_message}, 404 + + state = states[key] + if 'discovery_future' in state and state['discovery_future']: + state['discovery_future'].cancel() + del states[key] + + logger.info(f"Deleted {label} playlist state: {key}") + return {"success": True, "message": "Playlist deleted"}, 200 + except Exception as e: + logger.error(f"Error deleting {label} playlist: {e}") + return {"error": str(e)}, 500 diff --git a/tests/discovery/test_discovery_endpoints.py b/tests/discovery/test_discovery_endpoints.py index 0b78c1b4..2e346f84 100644 --- a/tests/discovery/test_discovery_endpoints.py +++ b/tests/discovery/test_discovery_endpoints.py @@ -8,7 +8,30 @@ the originals handled is exercised here. from __future__ import annotations -from core.discovery.endpoints import convert_results_to_spotify_tracks +import threading + +from core.discovery.endpoints import ( + convert_results_to_spotify_tracks, + cancel_sync, + delete_playlist_state, +) + + +class _FakeFuture: + def __init__(self): + self.cancelled = False + + def cancel(self): + self.cancelled = True + + +def _cancel_infra(): + """Fresh sync infra (lock + the two shared dicts) for cancel_sync tests.""" + return { + 'sync_lock': threading.Lock(), + 'sync_states': {}, + 'active_sync_workers': {}, + } # --------------------------------------------------------------------------- @@ -125,3 +148,114 @@ def test_spotify_data_takes_precedence_over_auto_fields(): }] out = convert_results_to_spotify_tracks(results, 'Tidal') assert out[0]['id'] == 'D' + + +# --------------------------------------------------------------------------- +# cancel_sync +# --------------------------------------------------------------------------- + +def test_cancel_sync_not_found_returns_404(): + body, code = cancel_sync({}, 'missing', label='Tidal', + not_found_message='Tidal playlist not found', **_cancel_infra()) + assert code == 404 + assert body == {"error": "Tidal playlist not found"} + + +def test_cancel_sync_cancels_active_worker_and_reverts_state(): + infra = _cancel_infra() + infra['sync_states']['sp1'] = {"status": "running"} + infra['active_sync_workers']['sp1'] = _FakeFuture() + states = {'pl': {'phase': 'syncing', 'sync_playlist_id': 'sp1', + 'sync_progress': {'done': 1}, 'last_accessed': 0}} + + body, code = cancel_sync(states, 'pl', label='Tidal', + not_found_message='nf', **infra) + + assert code == 200 + assert body == {"success": True, "message": "Tidal sync cancelled"} + # sync marked cancelled, worker removed + assert infra['sync_states']['sp1'] == {"status": "cancelled"} + assert 'sp1' not in infra['active_sync_workers'] + # state reverted + assert states['pl']['phase'] == 'discovered' + assert states['pl']['sync_playlist_id'] is None + assert states['pl']['sync_progress'] == {} + assert states['pl']['last_accessed'] != 0 # touched + + +def test_cancel_sync_worker_absent_from_active_map_is_safe(): + infra = _cancel_infra() # active_sync_workers empty + states = {'pl': {'phase': 'syncing', 'sync_playlist_id': 'sp1'}} + body, code = cancel_sync(states, 'pl', label='Deezer', not_found_message='nf', **infra) + assert code == 200 + assert infra['sync_states']['sp1'] == {"status": "cancelled"} + + +def test_cancel_sync_no_sync_in_progress_still_reverts(): + infra = _cancel_infra() + states = {'pl': {'phase': 'discovered'}} # no sync_playlist_id + body, code = cancel_sync(states, 'pl', label='Qobuz', not_found_message='nf', **infra) + assert code == 200 + assert states['pl']['sync_playlist_id'] is None + assert states['pl']['sync_progress'] == {} + assert infra['sync_states'] == {} # nothing cancelled + + +def test_cancel_sync_label_in_message(): + infra = _cancel_infra() + states = {'pl': {}} + body, _ = cancel_sync(states, 'pl', label='iTunes Link', not_found_message='nf', **infra) + assert body["message"] == "iTunes Link sync cancelled" + + +def test_cancel_sync_exception_returns_500(): + infra = _cancel_infra() + states = {'pl': object()} # not subscriptable -> raises inside try + body, code = cancel_sync(states, 'pl', label='Tidal', not_found_message='nf', **infra) + assert code == 500 + assert "error" in body + + +# --------------------------------------------------------------------------- +# delete_playlist_state +# --------------------------------------------------------------------------- + +def test_delete_not_found_returns_404(): + body, code = delete_playlist_state({}, 'missing', label='Tidal', + not_found_message='Tidal playlist not found') + assert code == 404 + assert body == {"error": "Tidal playlist not found"} + + +def test_delete_cancels_discovery_future_and_removes_state(): + fut = _FakeFuture() + states = {'pl': {'discovery_future': fut}} + body, code = delete_playlist_state(states, 'pl', label='Tidal', not_found_message='nf') + assert code == 200 + assert body == {"success": True, "message": "Playlist deleted"} + assert fut.cancelled is True + assert 'pl' not in states + + +def test_delete_without_discovery_future_still_removes(): + states = {'pl': {'phase': 'discovered'}} # no discovery_future + body, code = delete_playlist_state(states, 'pl', label='Deezer', not_found_message='nf') + assert code == 200 + assert 'pl' not in states + + +def test_delete_falsy_discovery_future_not_cancelled(): + states = {'pl': {'discovery_future': None}} + body, code = delete_playlist_state(states, 'pl', label='Qobuz', not_found_message='nf') + assert code == 200 + assert 'pl' not in states + + +def test_delete_exception_returns_500(): + fut = object() # no .cancel() -> AttributeError inside try + states = {'pl': {'discovery_future': fut}} + body, code = delete_playlist_state(states, 'pl', label='Tidal', not_found_message='nf') + assert code == 500 + assert "error" in body + # state NOT deleted because the exception fired before del + assert 'pl' in states diff --git a/web_server.py b/web_server.py index 27742136..54d0a247 100644 --- a/web_server.py +++ b/web_server.py @@ -20802,25 +20802,7 @@ def reset_tidal_playlist(playlist_id): @app.route('/api/tidal/delete/', methods=['POST']) def delete_tidal_playlist(playlist_id): """Delete Tidal playlist state completely""" - try: - if playlist_id not in tidal_discovery_states: - return jsonify({"error": "Tidal playlist not found"}), 404 - - state = tidal_discovery_states[playlist_id] - - # Stop any active discovery - if 'discovery_future' in state and state['discovery_future']: - state['discovery_future'].cancel() - - # Remove from state dictionary - del tidal_discovery_states[playlist_id] - - logger.info(f"Deleted Tidal playlist state: {playlist_id}") - return jsonify({"success": True, "message": "Playlist deleted"}) - - except Exception as e: - logger.error(f"Error deleting Tidal playlist: {e}") - return jsonify({"error": str(e)}), 500 + return _delete_source_playlist(tidal_discovery_states, playlist_id, "Tidal", "Tidal playlist not found") @app.route('/api/tidal/update_phase/', methods=['POST']) def update_tidal_playlist_phase(playlist_id): @@ -21074,7 +21056,31 @@ from core.discovery.scoring import ( # Tidal discovery worker logic lives in core/discovery/tidal.py. from core.discovery import tidal as _discovery_tidal # Source-agnostic discovery route helpers (lifted from the per-source copies). -from core.discovery.endpoints import convert_results_to_spotify_tracks +from core.discovery.endpoints import ( + convert_results_to_spotify_tracks, + cancel_sync as _cancel_sync_core, + delete_playlist_state as _delete_playlist_state_core, +) + + +def _cancel_source_sync(states, key, label, not_found_message): + """Thin glue: wire web_server's sync infra into the lifted cancel_sync + helper and jsonify the result. Used by the per-source cancel routes.""" + body, code = _cancel_sync_core( + states, key, label=label, not_found_message=not_found_message, + sync_lock=sync_lock, sync_states=sync_states, + active_sync_workers=active_sync_workers, + ) + return jsonify(body), code + + +def _delete_source_playlist(states, key, label, not_found_message): + """Thin glue for the per-source delete routes that share the identical + delete body (Tidal/Deezer/Qobuz/Spotify-Public).""" + body, code = _delete_playlist_state_core( + states, key, label=label, not_found_message=not_found_message, + ) + return jsonify(body), code def _build_tidal_discovery_deps(): @@ -21214,33 +21220,7 @@ def get_tidal_sync_status(playlist_id): @app.route('/api/tidal/sync/cancel/', methods=['POST']) def cancel_tidal_sync(playlist_id): """Cancel sync for a Tidal playlist""" - try: - if playlist_id not in tidal_discovery_states: - return jsonify({"error": "Tidal playlist not found"}), 404 - - state = tidal_discovery_states[playlist_id] - state['last_accessed'] = time.time() # Update access time - sync_playlist_id = state.get('sync_playlist_id') - - if sync_playlist_id: - # Cancel the sync using existing sync infrastructure - with sync_lock: - sync_states[sync_playlist_id] = {"status": "cancelled"} - - # Clean up sync worker - if sync_playlist_id in active_sync_workers: - del active_sync_workers[sync_playlist_id] - - # Revert Tidal state - state['phase'] = 'discovered' - state['sync_playlist_id'] = None - state['sync_progress'] = {} - - return jsonify({"success": True, "message": "Tidal sync cancelled"}) - - except Exception as e: - logger.error(f"Error cancelling Tidal sync: {e}") - return jsonify({"error": str(e)}), 500 + return _cancel_source_sync(tidal_discovery_states, playlist_id, "Tidal", "Tidal playlist not found") # =================================================================== @@ -21699,25 +21679,7 @@ def reset_deezer_playlist(playlist_id): @app.route('/api/deezer/delete/', methods=['POST']) def delete_deezer_playlist(playlist_id): """Delete Deezer playlist state completely""" - try: - if playlist_id not in deezer_discovery_states: - return jsonify({"error": "Deezer playlist not found"}), 404 - - state = deezer_discovery_states[playlist_id] - - # Stop any active discovery - if 'discovery_future' in state and state['discovery_future']: - state['discovery_future'].cancel() - - # Remove from state dictionary - del deezer_discovery_states[playlist_id] - - logger.info(f"Deleted Deezer playlist state: {playlist_id}") - return jsonify({"success": True, "message": "Playlist deleted"}) - - except Exception as e: - logger.error(f"Error deleting Deezer playlist: {e}") - return jsonify({"error": str(e)}), 500 + return _delete_source_playlist(deezer_discovery_states, playlist_id, "Deezer", "Deezer playlist not found") @app.route('/api/deezer/update_phase/', methods=['POST']) def update_deezer_playlist_phase(playlist_id): @@ -21893,33 +21855,7 @@ def get_deezer_sync_status(playlist_id): @app.route('/api/deezer/sync/cancel/', methods=['POST']) def cancel_deezer_sync(playlist_id): """Cancel sync for a Deezer playlist""" - try: - if playlist_id not in deezer_discovery_states: - return jsonify({"error": "Deezer playlist not found"}), 404 - - state = deezer_discovery_states[playlist_id] - state['last_accessed'] = time.time() - sync_playlist_id = state.get('sync_playlist_id') - - if sync_playlist_id: - # Cancel the sync using existing sync infrastructure - with sync_lock: - sync_states[sync_playlist_id] = {"status": "cancelled"} - - # Clean up sync worker - if sync_playlist_id in active_sync_workers: - del active_sync_workers[sync_playlist_id] - - # Revert Deezer state - state['phase'] = 'discovered' - state['sync_playlist_id'] = None - state['sync_progress'] = {} - - return jsonify({"success": True, "message": "Deezer sync cancelled"}) - - except Exception as e: - logger.error(f"Error cancelling Deezer sync: {e}") - return jsonify({"error": str(e)}), 500 + return _cancel_source_sync(deezer_discovery_states, playlist_id, "Deezer", "Deezer playlist not found") # =================================================================== @@ -22335,23 +22271,7 @@ def reset_qobuz_playlist(playlist_id): @app.route('/api/qobuz/delete/', methods=['POST']) def delete_qobuz_playlist(playlist_id): """Delete Qobuz playlist state completely.""" - try: - if playlist_id not in qobuz_discovery_states: - return jsonify({"error": "Qobuz playlist not found"}), 404 - - state = qobuz_discovery_states[playlist_id] - - if 'discovery_future' in state and state['discovery_future']: - state['discovery_future'].cancel() - - del qobuz_discovery_states[playlist_id] - - logger.info(f"Deleted Qobuz playlist state: {playlist_id}") - return jsonify({"success": True, "message": "Playlist deleted"}) - - except Exception as e: - logger.error(f"Error deleting Qobuz playlist: {e}") - return jsonify({"error": str(e)}), 500 + return _delete_source_playlist(qobuz_discovery_states, playlist_id, "Qobuz", "Qobuz playlist not found") @app.route('/api/qobuz/update_phase/', methods=['POST']) @@ -22517,29 +22437,7 @@ def get_qobuz_sync_status(playlist_id): @app.route('/api/qobuz/sync/cancel/', methods=['POST']) def cancel_qobuz_sync(playlist_id): """Cancel sync for a Qobuz playlist.""" - try: - if playlist_id not in qobuz_discovery_states: - return jsonify({"error": "Qobuz playlist not found"}), 404 - - state = qobuz_discovery_states[playlist_id] - state['last_accessed'] = time.time() - sync_playlist_id = state.get('sync_playlist_id') - - if sync_playlist_id: - with sync_lock: - sync_states[sync_playlist_id] = {"status": "cancelled"} - if sync_playlist_id in active_sync_workers: - del active_sync_workers[sync_playlist_id] - - state['phase'] = 'discovered' - state['sync_playlist_id'] = None - state['sync_progress'] = {} - - return jsonify({"success": True, "message": "Qobuz sync cancelled"}) - - except Exception as e: - logger.error(f"Error cancelling Qobuz sync: {e}") - return jsonify({"error": str(e)}), 500 + return _cancel_source_sync(qobuz_discovery_states, playlist_id, "Qobuz", "Qobuz playlist not found") # =================================================================== @@ -23239,25 +23137,7 @@ def reset_spotify_public_playlist(url_hash): @app.route('/api/spotify-public/delete/', methods=['POST']) def delete_spotify_public_playlist(url_hash): """Delete Spotify Public playlist state completely""" - try: - if url_hash not in spotify_public_discovery_states: - return jsonify({"error": "Spotify Public playlist not found"}), 404 - - state = spotify_public_discovery_states[url_hash] - - # Stop any active discovery - if 'discovery_future' in state and state['discovery_future']: - state['discovery_future'].cancel() - - # Remove from state dictionary - del spotify_public_discovery_states[url_hash] - - logger.info(f"Deleted Spotify Public playlist state: {url_hash}") - return jsonify({"success": True, "message": "Playlist deleted"}) - - except Exception as e: - logger.error(f"Error deleting Spotify Public playlist: {e}") - return jsonify({"error": str(e)}), 500 + return _delete_source_playlist(spotify_public_discovery_states, url_hash, "Spotify Public", "Spotify Public playlist not found") @app.route('/api/spotify-public/update_phase/', methods=['POST']) def update_spotify_public_playlist_phase(url_hash): @@ -23434,33 +23314,7 @@ def get_spotify_public_sync_status(url_hash): @app.route('/api/spotify-public/sync/cancel/', methods=['POST']) def cancel_spotify_public_sync(url_hash): """Cancel sync for a Spotify Public playlist""" - try: - if url_hash not in spotify_public_discovery_states: - return jsonify({"error": "Spotify Public playlist not found"}), 404 - - state = spotify_public_discovery_states[url_hash] - state['last_accessed'] = time.time() - sync_playlist_id = state.get('sync_playlist_id') - - if sync_playlist_id: - # Cancel the sync using existing sync infrastructure - with sync_lock: - sync_states[sync_playlist_id] = {"status": "cancelled"} - - # Clean up sync worker - if sync_playlist_id in active_sync_workers: - del active_sync_workers[sync_playlist_id] - - # Revert Spotify Public state - state['phase'] = 'discovered' - state['sync_playlist_id'] = None - state['sync_progress'] = {} - - return jsonify({"success": True, "message": "Spotify Public sync cancelled"}) - - except Exception as e: - logger.error(f"Error cancelling Spotify Public sync: {e}") - return jsonify({"error": str(e)}), 500 + return _cancel_source_sync(spotify_public_discovery_states, url_hash, "Spotify Public", "Spotify Public playlist not found") # =================================================================== @@ -23890,24 +23744,7 @@ def get_itunes_link_sync_status(url_hash): @app.route('/api/itunes-link/sync/cancel/', methods=['POST']) def cancel_itunes_link_sync(url_hash): - try: - if url_hash not in itunes_link_discovery_states: - return jsonify({"error": "iTunes Link not found"}), 404 - state = itunes_link_discovery_states[url_hash] - state['last_accessed'] = time.time() - sync_playlist_id = state.get('sync_playlist_id') - if sync_playlist_id: - with sync_lock: - sync_states[sync_playlist_id] = {"status": "cancelled"} - if sync_playlist_id in active_sync_workers: - del active_sync_workers[sync_playlist_id] - state['phase'] = 'discovered' - state['sync_playlist_id'] = None - state['sync_progress'] = {} - return jsonify({"success": True, "message": "iTunes Link sync cancelled"}) - except Exception as e: - logger.error(f"Error cancelling iTunes Link sync: {e}") - return jsonify({"error": str(e)}), 500 + return _cancel_source_sync(itunes_link_discovery_states, url_hash, "iTunes Link", "iTunes Link not found") # =================================================================== @@ -24555,33 +24392,7 @@ def get_youtube_sync_status(url_hash): @app.route('/api/youtube/sync/cancel/', methods=['POST']) def cancel_youtube_sync(url_hash): """Cancel sync for a YouTube playlist""" - try: - if url_hash not in youtube_playlist_states: - return jsonify({"error": "YouTube playlist not found"}), 404 - - state = youtube_playlist_states[url_hash] - state['last_accessed'] = time.time() # Update access time - sync_playlist_id = state.get('sync_playlist_id') - - if sync_playlist_id: - # Cancel the sync using existing sync infrastructure - with sync_lock: - sync_states[sync_playlist_id] = {"status": "cancelled"} - - # Clean up sync worker - if sync_playlist_id in active_sync_workers: - del active_sync_workers[sync_playlist_id] - - # Revert YouTube state - state['phase'] = 'discovered' - state['sync_playlist_id'] = None - state['sync_progress'] = {} - - return jsonify({"success": True, "message": "YouTube sync cancelled"}) - - except Exception as e: - logger.error(f"Error cancelling YouTube sync: {e}") - return jsonify({"error": str(e)}), 500 + return _cancel_source_sync(youtube_playlist_states, url_hash, "YouTube", "YouTube playlist not found") # New YouTube Playlist Management Endpoints (for persistent state) @@ -30690,34 +30501,7 @@ def get_listenbrainz_sync_status(playlist_mbid): @app.route('/api/listenbrainz/sync/cancel/', methods=['POST']) def cancel_listenbrainz_sync(playlist_mbid): """Cancel sync for a ListenBrainz playlist""" - try: - state_key = _lb_state_key(playlist_mbid) - if state_key not in listenbrainz_playlist_states: - return jsonify({"error": "ListenBrainz playlist not found"}), 404 - - state = listenbrainz_playlist_states[state_key] - state['last_accessed'] = time.time() # Update access time - sync_playlist_id = state.get('sync_playlist_id') - - if sync_playlist_id: - # Cancel the sync using existing sync infrastructure - with sync_lock: - sync_states[sync_playlist_id] = {"status": "cancelled"} - - # Clean up sync worker - if sync_playlist_id in active_sync_workers: - del active_sync_workers[sync_playlist_id] - - # Revert ListenBrainz state - state['phase'] = 'discovered' - state['sync_playlist_id'] = None - state['sync_progress'] = {} - - return jsonify({"success": True, "message": "ListenBrainz sync cancelled"}) - - except Exception as e: - logger.error(f"Error cancelling ListenBrainz sync: {e}") - return jsonify({"error": str(e)}), 500 + return _cancel_source_sync(listenbrainz_playlist_states, _lb_state_key(playlist_mbid), "ListenBrainz", "ListenBrainz playlist not found") @app.route('/api/metadata/start', methods=['POST']) def start_metadata_update(): From aad1d2b8f0e2debb9073c4c99c11d95b4bf694ff Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 16:51:16 -0700 Subject: [PATCH 12/52] Discovery lift (3/N): get_*_sync_status -> shared helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third cluster: the get__sync_status routes (Tidal, Deezer, Qobuz, Spotify-Public, iTunes-Link, YouTube, ListenBrainz) -> core.discovery. endpoints.get_sync_status(...), wired via _get_source_sync_status glue. This cluster carries the real per-source quirks, all captured 1:1 as params: - not_found_message (iTunes-Link uses "iTunes Link not found"). - error_label vs activity_subject — these DIFFER for Spotify-Public: the activity feed says "Spotify Link playlist ..." while the except log says "Error getting Spotify Public sync status". - playlist-name accessor, three styles lifted verbatim as named helpers: playlist_name_attr_or_unknown (Tidal: object .name), playlist_name_strict (Deezer/Qobuz/Spotify-Public/iTunes: state['playlist']['name'], can raise), playlist_name_safe (YouTube/ListenBrainz: .get default). The strict getter preserves the original's behavior of raising -> 500 AFTER phase/sync_progress were already mutated. - ListenBrainz key via _lb_state_key (caller-resolved). Beatport stays separate (different payload: status not sync_status, sync_id, no lock, chart key). Tests: +9 (3 name accessors incl. raise/fallback semantics; status 404s, running-no-mutation, finished+activity, error+revert+activity, and strict- getter-missing -> 500 after partial mutation). Full discovery suite: 171 passed. web_server.py: -244 lines. --- core/discovery/endpoints.py | 92 +++++++ tests/discovery/test_discovery_endpoints.py | 135 +++++++++ web_server.py | 290 ++------------------ 3 files changed, 250 insertions(+), 267 deletions(-) diff --git a/core/discovery/endpoints.py b/core/discovery/endpoints.py index 61d8cc0b..f9ca30b5 100644 --- a/core/discovery/endpoints.py +++ b/core/discovery/endpoints.py @@ -157,3 +157,95 @@ def delete_playlist_state( except Exception as e: logger.error(f"Error deleting {label} playlist: {e}") return {"error": str(e)}, 500 + + +# --- playlist-name accessors ------------------------------------------------- +# The per-source sync-status handlers read the display name three different +# ways. Each is reproduced verbatim so the 1:1 behavior (including which ones +# raise vs. fall back to 'Unknown Playlist') is preserved. + +def playlist_name_attr_or_unknown(state: Dict[str, Any]) -> str: + """Tidal: playlist is an object — use ``.name`` or 'Unknown Playlist'.""" + pl = state.get('playlist') + return pl.name if pl and hasattr(pl, 'name') else 'Unknown Playlist' + + +def playlist_name_strict(state: Dict[str, Any]) -> str: + """Deezer / Qobuz / Spotify-Public / iTunes-Link: strict dict access — + raises (→ 500) if 'playlist' is missing, exactly like the originals.""" + return state['playlist']['name'] + + +def playlist_name_safe(state: Dict[str, Any]) -> str: + """YouTube / ListenBrainz: safe dict access, defaulting to 'Unknown + Playlist'.""" + return state.get('playlist', {}).get('name', 'Unknown Playlist') + + +def get_sync_status( + states: Dict[str, Any], + key: str, + *, + not_found_message: str, + error_label: str, + activity_subject: str, + playlist_name_getter, + sync_lock: Any, + sync_states: Dict[str, Any], + add_activity_item, +) -> Tuple[Dict[str, Any], int]: + """Report sync status for one discovery playlist, posting an activity-feed + item when the sync finishes or errors. + + 1:1 lift of the ``get__sync_status`` bodies (Tidal, Deezer, Qobuz, + Spotify-Public, iTunes-Link, YouTube, ListenBrainz). Per-source variation + is captured by the parameters: + + - ``not_found_message`` — the 404 string (iTunes-Link drops "playlist"). + - ``error_label`` — used in the except log ("Error getting sync status"). + - ``activity_subject`` — the activity-feed prefix; note Spotify-Public uses + "Spotify Link playlist" while its error_label is "Spotify Public". + - ``playlist_name_getter`` — one of the accessors above (attr/strict/safe); + the strict one can raise, matching the originals (→ 500). The state's + phase/sync_progress are mutated BEFORE the name is read, so a raising + getter leaves the same partial mutation the original did. + + Beatport is NOT routed here — it returns a different payload (``status`` + not ``sync_status``, includes ``sync_id``, no lock, ``chart`` key). + """ + try: + if key not in states: + return {"error": not_found_message}, 404 + + state = states[key] + state['last_accessed'] = time.time() + sync_playlist_id = state.get('sync_playlist_id') + + if not sync_playlist_id: + return {"error": "No sync in progress"}, 404 + + with sync_lock: + sync_state = sync_states.get(sync_playlist_id, {}) + + response = { + 'phase': state['phase'], + 'sync_status': sync_state.get('status', 'unknown'), + 'progress': sync_state.get('progress', {}), + 'complete': sync_state.get('status') == 'finished', + 'error': sync_state.get('error'), + } + + if sync_state.get('status') == 'finished': + state['phase'] = 'sync_complete' + state['sync_progress'] = sync_state.get('progress', {}) + playlist_name = playlist_name_getter(state) + add_activity_item("", "Sync Complete", f"{activity_subject} '{playlist_name}' synced successfully", "Now") + elif sync_state.get('status') == 'error': + state['phase'] = 'discovered' + playlist_name = playlist_name_getter(state) + add_activity_item("", "Sync Failed", f"{activity_subject} '{playlist_name}' sync failed", "Now") + + return response, 200 + except Exception as e: + logger.error(f"Error getting {error_label} sync status: {e}") + return {"error": str(e)}, 500 diff --git a/tests/discovery/test_discovery_endpoints.py b/tests/discovery/test_discovery_endpoints.py index 2e346f84..968d79cf 100644 --- a/tests/discovery/test_discovery_endpoints.py +++ b/tests/discovery/test_discovery_endpoints.py @@ -14,6 +14,9 @@ from core.discovery.endpoints import ( convert_results_to_spotify_tracks, cancel_sync, delete_playlist_state, + get_sync_status, + playlist_name_strict as _pl_name_strict, + playlist_name_safe as _pl_name_safe, ) @@ -259,3 +262,135 @@ def test_delete_exception_returns_500(): assert "error" in body # state NOT deleted because the exception fired before del assert 'pl' in states + + +# --------------------------------------------------------------------------- +# playlist-name accessors +# --------------------------------------------------------------------------- + +class _Obj: + def __init__(self, name): + self.name = name + + +def test_name_attr_or_unknown(): + from core.discovery.endpoints import playlist_name_attr_or_unknown as g + assert g({'playlist': _Obj('My PL')}) == 'My PL' + assert g({'playlist': {'name': 'dict'}}) == 'Unknown Playlist' # dict has no .name attr + assert g({}) == 'Unknown Playlist' + assert g({'playlist': None}) == 'Unknown Playlist' + + +def test_name_strict(): + from core.discovery.endpoints import playlist_name_strict as g + assert g({'playlist': {'name': 'X'}}) == 'X' + import pytest + with pytest.raises(KeyError): + g({}) # strict -> raises, matching originals + + +def test_name_safe(): + from core.discovery.endpoints import playlist_name_safe as g + assert g({'playlist': {'name': 'X'}}) == 'X' + assert g({}) == 'Unknown Playlist' + assert g({'playlist': {}}) == 'Unknown Playlist' + + +# --------------------------------------------------------------------------- +# get_sync_status +# --------------------------------------------------------------------------- + +def _activity_recorder(): + calls = [] + + def add_activity_item(user, action, desc, when): + calls.append((user, action, desc, when)) + + return calls, add_activity_item + + +def _status_kwargs(infra, add_activity_item, *, not_found_message='nf', + error_label='Tidal', activity_subject='Tidal playlist', + name_getter=None): + return dict( + not_found_message=not_found_message, error_label=error_label, + activity_subject=activity_subject, + playlist_name_getter=name_getter or _pl_name_safe, + add_activity_item=add_activity_item, + sync_lock=infra['sync_lock'], sync_states=infra['sync_states'], + ) + + +def test_status_not_found(): + infra = _cancel_infra() + calls, add = _activity_recorder() + body, code = get_sync_status({}, 'missing', + **_status_kwargs(infra, add, not_found_message='Tidal playlist not found')) + assert code == 404 and body == {"error": "Tidal playlist not found"} + assert calls == [] + + +def test_status_no_sync_in_progress(): + infra = _cancel_infra() + calls, add = _activity_recorder() + states = {'pl': {'phase': 'discovered'}} # no sync_playlist_id + body, code = get_sync_status(states, 'pl', **_status_kwargs(infra, add)) + assert code == 404 and body == {"error": "No sync in progress"} + + +def test_status_running_returns_shape_without_mutation_or_activity(): + infra = _cancel_infra() + infra['sync_states']['sp1'] = {"status": "running", "progress": {"n": 2}} + calls, add = _activity_recorder() + states = {'pl': {'phase': 'syncing', 'sync_playlist_id': 'sp1'}} + body, code = get_sync_status(states, 'pl', **_status_kwargs(infra, add)) + assert code == 200 + assert body == { + 'phase': 'syncing', 'sync_status': 'running', + 'progress': {"n": 2}, 'complete': False, 'error': None, + } + assert states['pl']['phase'] == 'syncing' # unchanged + assert calls == [] + + +def test_status_finished_sets_complete_and_posts_activity(): + infra = _cancel_infra() + infra['sync_states']['sp1'] = {"status": "finished", "progress": {"done": 9}} + calls, add = _activity_recorder() + states = {'pl': {'phase': 'syncing', 'sync_playlist_id': 'sp1', + 'playlist': {'name': 'Mix'}}} + body, code = get_sync_status(states, 'pl', **_status_kwargs( + infra, add, activity_subject='Spotify Link playlist', + name_getter=_pl_name_strict)) + assert code == 200 + assert body['complete'] is True + assert states['pl']['phase'] == 'sync_complete' + assert states['pl']['sync_progress'] == {"done": 9} + assert calls == [("", "Sync Complete", "Spotify Link playlist 'Mix' synced successfully", "Now")] + + +def test_status_error_reverts_and_posts_failed_activity(): + infra = _cancel_infra() + infra['sync_states']['sp1'] = {"status": "error", "error": "boom"} + calls, add = _activity_recorder() + states = {'pl': {'phase': 'syncing', 'sync_playlist_id': 'sp1', + 'playlist': {'name': 'Mix'}}} + body, code = get_sync_status(states, 'pl', **_status_kwargs( + infra, add, activity_subject='YouTube playlist', name_getter=_pl_name_safe)) + assert code == 200 + assert body['error'] == "boom" + assert states['pl']['phase'] == 'discovered' + assert calls == [("", "Sync Failed", "YouTube playlist 'Mix' sync failed", "Now")] + + +def test_status_strict_getter_missing_playlist_raises_500_after_partial_mutation(): + infra = _cancel_infra() + infra['sync_states']['sp1'] = {"status": "finished", "progress": {}} + calls, add = _activity_recorder() + states = {'pl': {'phase': 'syncing', 'sync_playlist_id': 'sp1'}} # no 'playlist' + body, code = get_sync_status(states, 'pl', **_status_kwargs( + infra, add, error_label='Deezer', name_getter=_pl_name_strict)) + assert code == 500 and "error" in body + # phase was set to sync_complete BEFORE the strict getter raised (1:1). + assert states['pl']['phase'] == 'sync_complete' + assert calls == [] # activity never posted diff --git a/web_server.py b/web_server.py index 54d0a247..ce7f9dec 100644 --- a/web_server.py +++ b/web_server.py @@ -21060,6 +21060,10 @@ from core.discovery.endpoints import ( convert_results_to_spotify_tracks, cancel_sync as _cancel_sync_core, delete_playlist_state as _delete_playlist_state_core, + get_sync_status as _get_sync_status_core, + playlist_name_attr_or_unknown as _pl_name_attr_or_unknown, + playlist_name_strict as _pl_name_strict, + playlist_name_safe as _pl_name_safe, ) @@ -21083,6 +21087,18 @@ def _delete_source_playlist(states, key, label, not_found_message): return jsonify(body), code +def _get_source_sync_status(states, key, not_found_message, error_label, + activity_subject, name_getter): + """Thin glue for the per-source get_*_sync_status routes — wires the sync + infra + add_activity_item into the lifted helper and jsonifies.""" + body, code = _get_sync_status_core( + states, key, not_found_message=not_found_message, error_label=error_label, + activity_subject=activity_subject, playlist_name_getter=name_getter, + sync_lock=sync_lock, sync_states=sync_states, add_activity_item=add_activity_item, + ) + return jsonify(body), code + + def _build_tidal_discovery_deps(): """Build the TidalDiscoveryDeps bundle from web_server.py globals on each call.""" return _discovery_tidal.TidalDiscoveryDeps( @@ -21174,48 +21190,7 @@ def start_tidal_sync(playlist_id): @app.route('/api/tidal/sync/status/', methods=['GET']) def get_tidal_sync_status(playlist_id): """Get sync status for a Tidal playlist""" - try: - if playlist_id not in tidal_discovery_states: - return jsonify({"error": "Tidal playlist not found"}), 404 - - state = tidal_discovery_states[playlist_id] - state['last_accessed'] = time.time() # Update access time - sync_playlist_id = state.get('sync_playlist_id') - - if not sync_playlist_id: - return jsonify({"error": "No sync in progress"}), 404 - - # Get sync status from existing sync infrastructure - with sync_lock: - sync_state = sync_states.get(sync_playlist_id, {}) - - response = { - 'phase': state['phase'], - 'sync_status': sync_state.get('status', 'unknown'), - 'progress': sync_state.get('progress', {}), - 'complete': sync_state.get('status') == 'finished', - 'error': sync_state.get('error') - } - - # Update Tidal state if sync completed - if sync_state.get('status') == 'finished': - state['phase'] = 'sync_complete' - state['sync_progress'] = sync_state.get('progress', {}) - # Add activity for sync completion - playlist = state.get('playlist') - playlist_name = playlist.name if playlist and hasattr(playlist, 'name') else 'Unknown Playlist' - add_activity_item("", "Sync Complete", f"Tidal playlist '{playlist_name}' synced successfully", "Now") - elif sync_state.get('status') == 'error': - state['phase'] = 'discovered' # Revert on error - playlist = state.get('playlist') - playlist_name = playlist.name if playlist and hasattr(playlist, 'name') else 'Unknown Playlist' - add_activity_item("", "Sync Failed", f"Tidal playlist '{playlist_name}' sync failed", "Now") - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting Tidal sync status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_sync_status(tidal_discovery_states, playlist_id, "Tidal playlist not found", "Tidal", "Tidal playlist", _pl_name_attr_or_unknown) @app.route('/api/tidal/sync/cancel/', methods=['POST']) def cancel_tidal_sync(playlist_id): @@ -21812,45 +21787,7 @@ def start_deezer_sync(playlist_id): @app.route('/api/deezer/sync/status/', methods=['GET']) def get_deezer_sync_status(playlist_id): """Get sync status for a Deezer playlist""" - try: - if playlist_id not in deezer_discovery_states: - return jsonify({"error": "Deezer playlist not found"}), 404 - - state = deezer_discovery_states[playlist_id] - state['last_accessed'] = time.time() - sync_playlist_id = state.get('sync_playlist_id') - - if not sync_playlist_id: - return jsonify({"error": "No sync in progress"}), 404 - - # Get sync status from existing sync infrastructure - with sync_lock: - sync_state = sync_states.get(sync_playlist_id, {}) - - response = { - 'phase': state['phase'], - 'sync_status': sync_state.get('status', 'unknown'), - 'progress': sync_state.get('progress', {}), - 'complete': sync_state.get('status') == 'finished', - 'error': sync_state.get('error') - } - - # Update Deezer state if sync completed - if sync_state.get('status') == 'finished': - state['phase'] = 'sync_complete' - state['sync_progress'] = sync_state.get('progress', {}) - playlist_name = state['playlist']['name'] - add_activity_item("", "Sync Complete", f"Deezer playlist '{playlist_name}' synced successfully", "Now") - elif sync_state.get('status') == 'error': - state['phase'] = 'discovered' # Revert on error - playlist_name = state['playlist']['name'] - add_activity_item("", "Sync Failed", f"Deezer playlist '{playlist_name}' sync failed", "Now") - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting Deezer sync status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_sync_status(deezer_discovery_states, playlist_id, "Deezer playlist not found", "Deezer", "Deezer playlist", _pl_name_strict) @app.route('/api/deezer/sync/cancel/', methods=['POST']) def cancel_deezer_sync(playlist_id): @@ -22395,43 +22332,7 @@ def start_qobuz_sync(playlist_id): @app.route('/api/qobuz/sync/status/', methods=['GET']) def get_qobuz_sync_status(playlist_id): """Get sync status for a Qobuz playlist.""" - try: - if playlist_id not in qobuz_discovery_states: - return jsonify({"error": "Qobuz playlist not found"}), 404 - - state = qobuz_discovery_states[playlist_id] - state['last_accessed'] = time.time() - sync_playlist_id = state.get('sync_playlist_id') - - if not sync_playlist_id: - return jsonify({"error": "No sync in progress"}), 404 - - with sync_lock: - sync_state = sync_states.get(sync_playlist_id, {}) - - response = { - 'phase': state['phase'], - 'sync_status': sync_state.get('status', 'unknown'), - 'progress': sync_state.get('progress', {}), - 'complete': sync_state.get('status') == 'finished', - 'error': sync_state.get('error') - } - - if sync_state.get('status') == 'finished': - state['phase'] = 'sync_complete' - state['sync_progress'] = sync_state.get('progress', {}) - playlist_name = state['playlist']['name'] - add_activity_item("", "Sync Complete", f"Qobuz playlist '{playlist_name}' synced successfully", "Now") - elif sync_state.get('status') == 'error': - state['phase'] = 'discovered' - playlist_name = state['playlist']['name'] - add_activity_item("", "Sync Failed", f"Qobuz playlist '{playlist_name}' sync failed", "Now") - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting Qobuz sync status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_sync_status(qobuz_discovery_states, playlist_id, "Qobuz playlist not found", "Qobuz", "Qobuz playlist", _pl_name_strict) @app.route('/api/qobuz/sync/cancel/', methods=['POST']) @@ -23271,45 +23172,7 @@ def start_spotify_public_sync(url_hash): @app.route('/api/spotify-public/sync/status/', methods=['GET']) def get_spotify_public_sync_status(url_hash): """Get sync status for a Spotify Public playlist""" - try: - if url_hash not in spotify_public_discovery_states: - return jsonify({"error": "Spotify Public playlist not found"}), 404 - - state = spotify_public_discovery_states[url_hash] - state['last_accessed'] = time.time() - sync_playlist_id = state.get('sync_playlist_id') - - if not sync_playlist_id: - return jsonify({"error": "No sync in progress"}), 404 - - # Get sync status from existing sync infrastructure - with sync_lock: - sync_state = sync_states.get(sync_playlist_id, {}) - - response = { - 'phase': state['phase'], - 'sync_status': sync_state.get('status', 'unknown'), - 'progress': sync_state.get('progress', {}), - 'complete': sync_state.get('status') == 'finished', - 'error': sync_state.get('error') - } - - # Update Spotify Public state if sync completed - if sync_state.get('status') == 'finished': - state['phase'] = 'sync_complete' - state['sync_progress'] = sync_state.get('progress', {}) - playlist_name = state['playlist']['name'] - add_activity_item("", "Sync Complete", f"Spotify Link playlist '{playlist_name}' synced successfully", "Now") - elif sync_state.get('status') == 'error': - state['phase'] = 'discovered' # Revert on error - playlist_name = state['playlist']['name'] - add_activity_item("", "Sync Failed", f"Spotify Link playlist '{playlist_name}' sync failed", "Now") - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting Spotify Public sync status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_sync_status(spotify_public_discovery_states, url_hash, "Spotify Public playlist not found", "Spotify Public", "Spotify Link playlist", _pl_name_strict) @app.route('/api/spotify-public/sync/cancel/', methods=['POST']) def cancel_spotify_public_sync(url_hash): @@ -23711,35 +23574,7 @@ def start_itunes_link_sync(url_hash): @app.route('/api/itunes-link/sync/status/', methods=['GET']) def get_itunes_link_sync_status(url_hash): - try: - if url_hash not in itunes_link_discovery_states: - return jsonify({"error": "iTunes Link not found"}), 404 - state = itunes_link_discovery_states[url_hash] - state['last_accessed'] = time.time() - sync_playlist_id = state.get('sync_playlist_id') - if not sync_playlist_id: - return jsonify({"error": "No sync in progress"}), 404 - - with sync_lock: - sync_state = sync_states.get(sync_playlist_id, {}) - response = { - 'phase': state['phase'], - 'sync_status': sync_state.get('status', 'unknown'), - 'progress': sync_state.get('progress', {}), - 'complete': sync_state.get('status') == 'finished', - 'error': sync_state.get('error') - } - if sync_state.get('status') == 'finished': - state['phase'] = 'sync_complete' - state['sync_progress'] = sync_state.get('progress', {}) - add_activity_item("", "Sync Complete", f"iTunes Link '{state['playlist']['name']}' synced successfully", "Now") - elif sync_state.get('status') == 'error': - state['phase'] = 'discovered' - add_activity_item("", "Sync Failed", f"iTunes Link '{state['playlist']['name']}' sync failed", "Now") - return jsonify(response) - except Exception as e: - logger.error(f"Error getting iTunes Link sync status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_sync_status(itunes_link_discovery_states, url_hash, "iTunes Link not found", "iTunes Link", "iTunes Link", _pl_name_strict) @app.route('/api/itunes-link/sync/cancel/', methods=['POST']) @@ -24348,46 +24183,7 @@ def start_youtube_sync(url_hash): @app.route('/api/youtube/sync/status/', methods=['GET']) def get_youtube_sync_status(url_hash): """Get sync status for a YouTube playlist""" - try: - if url_hash not in youtube_playlist_states: - return jsonify({"error": "YouTube playlist not found"}), 404 - - state = youtube_playlist_states[url_hash] - state['last_accessed'] = time.time() # Update access time - sync_playlist_id = state.get('sync_playlist_id') - - if not sync_playlist_id: - return jsonify({"error": "No sync in progress"}), 404 - - # Get sync status from existing sync infrastructure - with sync_lock: - sync_state = sync_states.get(sync_playlist_id, {}) - - response = { - 'phase': state['phase'], - 'sync_status': sync_state.get('status', 'unknown'), - 'progress': sync_state.get('progress', {}), - 'complete': sync_state.get('status') == 'finished', - 'error': sync_state.get('error') - } - - # Update YouTube state if sync completed - if sync_state.get('status') == 'finished': - state['phase'] = 'sync_complete' - state['sync_progress'] = sync_state.get('progress', {}) - # Add activity for sync completion - playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist') - add_activity_item("", "Sync Complete", f"YouTube playlist '{playlist_name}' synced successfully", "Now") - elif sync_state.get('status') == 'error': - state['phase'] = 'discovered' # Revert on error - playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist') - add_activity_item("", "Sync Failed", f"YouTube playlist '{playlist_name}' sync failed", "Now") - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting YouTube sync status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_sync_status(youtube_playlist_states, url_hash, "YouTube playlist not found", "YouTube", "YouTube playlist", _pl_name_safe) @app.route('/api/youtube/sync/cancel/', methods=['POST']) def cancel_youtube_sync(url_hash): @@ -30456,47 +30252,7 @@ def start_listenbrainz_sync(playlist_mbid): @app.route('/api/listenbrainz/sync/status/', methods=['GET']) def get_listenbrainz_sync_status(playlist_mbid): """Get sync status for a ListenBrainz playlist""" - try: - state_key = _lb_state_key(playlist_mbid) - if state_key not in listenbrainz_playlist_states: - return jsonify({"error": "ListenBrainz playlist not found"}), 404 - - state = listenbrainz_playlist_states[state_key] - state['last_accessed'] = time.time() # Update access time - sync_playlist_id = state.get('sync_playlist_id') - - if not sync_playlist_id: - return jsonify({"error": "No sync in progress"}), 404 - - # Get sync status from existing sync infrastructure - with sync_lock: - sync_state = sync_states.get(sync_playlist_id, {}) - - response = { - 'phase': state['phase'], - 'sync_status': sync_state.get('status', 'unknown'), - 'progress': sync_state.get('progress', {}), - 'complete': sync_state.get('status') == 'finished', - 'error': sync_state.get('error') - } - - # Update ListenBrainz state if sync completed - if sync_state.get('status') == 'finished': - state['phase'] = 'sync_complete' - state['sync_progress'] = sync_state.get('progress', {}) - # Add activity for sync completion - playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist') - add_activity_item("", "Sync Complete", f"ListenBrainz playlist '{playlist_name}' synced successfully", "Now") - elif sync_state.get('status') == 'error': - state['phase'] = 'discovered' # Revert on error - playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist') - add_activity_item("", "Sync Failed", f"ListenBrainz playlist '{playlist_name}' sync failed", "Now") - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting ListenBrainz sync status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_sync_status(listenbrainz_playlist_states, _lb_state_key(playlist_mbid), "ListenBrainz playlist not found", "ListenBrainz", "ListenBrainz playlist", _pl_name_safe) @app.route('/api/listenbrainz/sync/cancel/', methods=['POST']) def cancel_listenbrainz_sync(playlist_mbid): From 8a9ed677ab668415c664102eec75513b6625f62a Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 17:00:20 -0700 Subject: [PATCH 13/52] Discovery lift (4/N): get_*_discovery_status -> shared helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourth cluster: get__discovery_status (all eight sources, Beatport included) -> core.discovery.endpoints.get_discovery_status(states, key, *, not_found_message, error_label), wired via _get_source_discovery_status. Unlike sync-status, the discovery-status response shape is byte-identical across every source (phase/status/progress/spotify_matches/spotify_total/ results/complete), so Beatport folds in here too. Only the 404 string ("... discovery not found" vs "... playlist not found" vs "Beatport chart not found") and the except-log label vary. ListenBrainz key via _lb_state_key. NOT touched this cluster: get_*_playlist_state (the sibling endpoints). Those genuinely diverge per source — different id-key name (playlist_id / url_hash / playlist_mbid), presence of url / created_at / download_process_id, Tidal's playlist.__dict__ serialization, and YouTube's strict (non-.get) field access. Folding them would need a flag pile that wouldn't be a clean 1:1, so they keep their own bodies. Tests: +4 (404, full response + last_accessed bump, complete=False when not 'discovered', missing-field -> 500). Full discovery suite: 175 passed. web_server.py: -155 lines. --- core/discovery/endpoints.py | 39 ++++ tests/discovery/test_discovery_endpoints.py | 51 ++++++ web_server.py | 189 ++------------------ 3 files changed, 107 insertions(+), 172 deletions(-) diff --git a/core/discovery/endpoints.py b/core/discovery/endpoints.py index f9ca30b5..81723df5 100644 --- a/core/discovery/endpoints.py +++ b/core/discovery/endpoints.py @@ -249,3 +249,42 @@ def get_sync_status( except Exception as e: logger.error(f"Error getting {error_label} sync status: {e}") return {"error": str(e)}, 500 + + +def get_discovery_status( + states: Dict[str, Any], + key: str, + *, + not_found_message: str, + error_label: str, +) -> Tuple[Dict[str, Any], int]: + """Report real-time discovery progress/results for one playlist. + + 1:1 lift of the byte-identical ``get__discovery_status`` bodies. + Unlike sync-status, this shape is identical for ALL eight sources — + Beatport included — so it folds in too. Only the 404 message + (".../discovery not found" vs ".../playlist not found" vs "Beatport chart + not found") and the except-log label vary, both passed in. The caller + resolves the key (ListenBrainz via ``_lb_state_key``). + + Returns ``(payload, status_code)``. + """ + try: + if key not in states: + return {"error": not_found_message}, 404 + + state = states[key] + state['last_accessed'] = time.time() + + return { + 'phase': state['phase'], + 'status': state['status'], + 'progress': state['discovery_progress'], + 'spotify_matches': state['spotify_matches'], + 'spotify_total': state['spotify_total'], + 'results': state['discovery_results'], + 'complete': state['phase'] == 'discovered', + }, 200 + except Exception as e: + logger.error(f"Error getting {error_label} discovery status: {e}") + return {"error": str(e)}, 500 diff --git a/tests/discovery/test_discovery_endpoints.py b/tests/discovery/test_discovery_endpoints.py index 968d79cf..846802f5 100644 --- a/tests/discovery/test_discovery_endpoints.py +++ b/tests/discovery/test_discovery_endpoints.py @@ -394,3 +394,54 @@ def test_status_strict_getter_missing_playlist_raises_500_after_partial_mutation # phase was set to sync_complete BEFORE the strict getter raised (1:1). assert states['pl']['phase'] == 'sync_complete' assert calls == [] # activity never posted + + +# --------------------------------------------------------------------------- +# get_discovery_status +# --------------------------------------------------------------------------- + +def test_discovery_status_not_found(): + from core.discovery.endpoints import get_discovery_status + body, code = get_discovery_status({}, 'missing', + not_found_message='Tidal discovery not found', + error_label='Tidal') + assert code == 404 and body == {"error": "Tidal discovery not found"} + + +def test_discovery_status_builds_response_and_bumps_access(): + from core.discovery.endpoints import get_discovery_status + state = { + 'phase': 'discovered', 'status': 'done', 'discovery_progress': 100, + 'spotify_matches': 8, 'spotify_total': 10, + 'discovery_results': [{'x': 1}], 'last_accessed': 0, + } + states = {'pl': state} + body, code = get_discovery_status(states, 'pl', + not_found_message='nf', error_label='Tidal') + assert code == 200 + assert body == { + 'phase': 'discovered', 'status': 'done', 'progress': 100, + 'spotify_matches': 8, 'spotify_total': 10, + 'results': [{'x': 1}], 'complete': True, + } + assert state['last_accessed'] != 0 + + +def test_discovery_status_complete_false_when_not_discovered(): + from core.discovery.endpoints import get_discovery_status + state = { + 'phase': 'discovering', 'status': 'running', 'discovery_progress': 40, + 'spotify_matches': 2, 'spotify_total': 10, 'discovery_results': [], + } + body, code = get_discovery_status({'pl': state}, 'pl', + not_found_message='nf', error_label='Deezer') + assert code == 200 and body['complete'] is False + + +def test_discovery_status_missing_field_raises_500(): + from core.discovery.endpoints import get_discovery_status + # state missing 'status' -> strict access raises -> 500 (matches original) + states = {'pl': {'phase': 'x'}} + body, code = get_discovery_status(states, 'pl', + not_found_message='nf', error_label='Beatport') + assert code == 500 and "error" in body diff --git a/web_server.py b/web_server.py index ce7f9dec..9ad29b8c 100644 --- a/web_server.py +++ b/web_server.py @@ -20560,28 +20560,7 @@ def start_tidal_discovery(playlist_id): @app.route('/api/tidal/discovery/status/', methods=['GET']) def get_tidal_discovery_status(playlist_id): """Get real-time discovery status for a Tidal playlist""" - try: - if playlist_id not in tidal_discovery_states: - return jsonify({"error": "Tidal discovery not found"}), 404 - - state = tidal_discovery_states[playlist_id] - state['last_accessed'] = time.time() # Update access time - - response = { - 'phase': state['phase'], - 'status': state['status'], - 'progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'results': state['discovery_results'], - 'complete': state['phase'] == 'discovered' - } - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting Tidal discovery status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_discovery_status(tidal_discovery_states, playlist_id, "Tidal discovery not found", "Tidal") @app.route('/api/tidal/discovery/update_match', methods=['POST']) @@ -21061,6 +21040,7 @@ from core.discovery.endpoints import ( cancel_sync as _cancel_sync_core, delete_playlist_state as _delete_playlist_state_core, get_sync_status as _get_sync_status_core, + get_discovery_status as _get_discovery_status_core, playlist_name_attr_or_unknown as _pl_name_attr_or_unknown, playlist_name_strict as _pl_name_strict, playlist_name_safe as _pl_name_safe, @@ -21099,6 +21079,14 @@ def _get_source_sync_status(states, key, not_found_message, error_label, return jsonify(body), code +def _get_source_discovery_status(states, key, not_found_message, error_label): + """Thin glue for the per-source get_*_discovery_status routes.""" + body, code = _get_discovery_status_core( + states, key, not_found_message=not_found_message, error_label=error_label, + ) + return jsonify(body), code + + def _build_tidal_discovery_deps(): """Build the TidalDiscoveryDeps bundle from web_server.py globals on each call.""" return _discovery_tidal.TidalDiscoveryDeps( @@ -21418,28 +21406,7 @@ def start_deezer_discovery(playlist_id): @app.route('/api/deezer/discovery/status/', methods=['GET']) def get_deezer_discovery_status(playlist_id): """Get real-time discovery status for a Deezer playlist""" - try: - if playlist_id not in deezer_discovery_states: - return jsonify({"error": "Deezer discovery not found"}), 404 - - state = deezer_discovery_states[playlist_id] - state['last_accessed'] = time.time() - - response = { - 'phase': state['phase'], - 'status': state['status'], - 'progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'results': state['discovery_results'], - 'complete': state['phase'] == 'discovered' - } - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting Deezer discovery status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_discovery_status(deezer_discovery_states, playlist_id, "Deezer discovery not found", "Deezer") @app.route('/api/deezer/discovery/update_match', methods=['POST']) def update_deezer_discovery_match(): @@ -21985,28 +21952,7 @@ def start_qobuz_discovery(playlist_id): @app.route('/api/qobuz/discovery/status/', methods=['GET']) def get_qobuz_discovery_status(playlist_id): """Get real-time discovery status for a Qobuz playlist.""" - try: - if playlist_id not in qobuz_discovery_states: - return jsonify({"error": "Qobuz discovery not found"}), 404 - - state = qobuz_discovery_states[playlist_id] - state['last_accessed'] = time.time() - - response = { - 'phase': state['phase'], - 'status': state['status'], - 'progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'results': state['discovery_results'], - 'complete': state['phase'] == 'discovered' - } - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting Qobuz discovery status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_discovery_status(qobuz_discovery_states, playlist_id, "Qobuz discovery not found", "Qobuz") @app.route('/api/qobuz/discovery/update_match', methods=['POST']) @@ -22803,28 +22749,7 @@ def start_spotify_public_discovery(url_hash): @app.route('/api/spotify-public/discovery/status/', methods=['GET']) def get_spotify_public_discovery_status(url_hash): """Get real-time discovery status for a Spotify Public playlist""" - try: - if url_hash not in spotify_public_discovery_states: - return jsonify({"error": "Spotify Public discovery not found"}), 404 - - state = spotify_public_discovery_states[url_hash] - state['last_accessed'] = time.time() - - response = { - 'phase': state['phase'], - 'status': state['status'], - 'progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'results': state['discovery_results'], - 'complete': state['phase'] == 'discovered' - } - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting Spotify Public discovery status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_discovery_status(spotify_public_discovery_states, url_hash, "Spotify Public discovery not found", "Spotify Public") @app.route('/api/spotify-public/discovery/update_match', methods=['POST']) def update_spotify_public_discovery_match(): @@ -23311,23 +23236,7 @@ def start_itunes_link_discovery(url_hash): @app.route('/api/itunes-link/discovery/status/', methods=['GET']) def get_itunes_link_discovery_status(url_hash): - try: - if url_hash not in itunes_link_discovery_states: - return jsonify({"error": "iTunes Link discovery not found"}), 404 - state = itunes_link_discovery_states[url_hash] - state['last_accessed'] = time.time() - return jsonify({ - 'phase': state['phase'], - 'status': state['status'], - 'progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'results': state['discovery_results'], - 'complete': state['phase'] == 'discovered' - }) - except Exception as e: - logger.error(f"Error getting iTunes Link discovery status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_discovery_status(itunes_link_discovery_states, url_hash, "iTunes Link discovery not found", "iTunes Link") def _update_itunes_link_discovery_result(identifier, track_index, spotify_track): @@ -23727,28 +23636,7 @@ def start_youtube_discovery(url_hash): @app.route('/api/youtube/discovery/status/', methods=['GET']) def get_youtube_discovery_status(url_hash): """Get real-time discovery status for a YouTube playlist""" - try: - if url_hash not in youtube_playlist_states: - return jsonify({"error": "YouTube playlist not found"}), 404 - - state = youtube_playlist_states[url_hash] - state['last_accessed'] = time.time() # Update access time - - response = { - 'phase': state['phase'], - 'status': state['status'], - 'progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'results': state['discovery_results'], - 'complete': state['phase'] == 'discovered' - } - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting YouTube discovery status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_discovery_status(youtube_playlist_states, url_hash, "YouTube playlist not found", "YouTube") @app.route('/api/youtube/discovery/unmatch', methods=['POST']) @@ -30002,29 +29890,7 @@ def start_listenbrainz_discovery(playlist_mbid): @app.route('/api/listenbrainz/discovery/status/', methods=['GET']) def get_listenbrainz_discovery_status(playlist_mbid): """Get real-time discovery status for a ListenBrainz playlist""" - try: - state_key = _lb_state_key(playlist_mbid) - if state_key not in listenbrainz_playlist_states: - return jsonify({"error": "ListenBrainz playlist not found"}), 404 - - state = listenbrainz_playlist_states[state_key] - state['last_accessed'] = time.time() - - response = { - 'phase': state['phase'], - 'status': state['status'], - 'progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'results': state['discovery_results'], - 'complete': state['phase'] == 'discovered' - } - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting ListenBrainz discovery status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_discovery_status(listenbrainz_playlist_states, _lb_state_key(playlist_mbid), "ListenBrainz playlist not found", "ListenBrainz") @app.route('/api/listenbrainz/update-phase/', methods=['POST']) def update_listenbrainz_phase(playlist_mbid): @@ -32018,28 +31884,7 @@ def start_beatport_discovery(url_hash): @app.route('/api/beatport/discovery/status/', methods=['GET']) def get_beatport_discovery_status(url_hash): """Get real-time discovery status for a Beatport chart""" - try: - if url_hash not in beatport_chart_states: - return jsonify({"error": "Beatport chart not found"}), 404 - - state = beatport_chart_states[url_hash] - state['last_accessed'] = time.time() - - response = { - 'phase': state['phase'], - 'status': state['status'], - 'progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'results': state['discovery_results'], - 'complete': state['phase'] == 'discovered' - } - - return jsonify(response) - - except Exception as e: - logger.error(f"Error getting Beatport discovery status: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_discovery_status(beatport_chart_states, url_hash, "Beatport chart not found", "Beatport") @app.route('/api/beatport/discovery/update_match', methods=['POST']) From 44b032b6c019b49bc8186aa7277864735b2cb411 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 17:15:13 -0700 Subject: [PATCH 14/52] Discovery lift (5/N): reset_*_playlist -> shared helper Fifth cluster: reset__playlist for the four sources with byte- identical bodies (Tidal, Deezer, Qobuz, Spotify-Public) -> core.discovery.endpoints.reset_playlist(states, key, *, label, not_found_message), wired via _reset_source_playlist. Resets phase/status to 'fresh', clears discovery/sync fields, cancels any discovery_future, and preserves the original playlist payload. Left with their own bodies (genuinely divergent): - YouTube: status -> 'parsed' (not 'fresh'), no download_process_id, logs the playlist name, "reset to fresh state". - ListenBrainz: status -> 'cached', logs playlist title, returns {"success": True, "phase": "fresh"} (different payload), _lb_state_key. - iTunes-Link: state.update(...), no info log, "iTunes Link reset to fresh phase". Tests: +4 (404, full clear + playlist preserved + future cancelled, no-future path, exception -> 500). Full discovery suite: 179 passed. web_server.py: -100 lines. --- core/discovery/endpoints.py | 46 +++++++ tests/discovery/test_discovery_endpoints.py | 60 +++++++++ web_server.py | 128 +++----------------- 3 files changed, 120 insertions(+), 114 deletions(-) diff --git a/core/discovery/endpoints.py b/core/discovery/endpoints.py index 81723df5..038dd889 100644 --- a/core/discovery/endpoints.py +++ b/core/discovery/endpoints.py @@ -288,3 +288,49 @@ def get_discovery_status( except Exception as e: logger.error(f"Error getting {error_label} discovery status: {e}") return {"error": str(e)}, 500 + + +def reset_playlist( + states: Dict[str, Any], + key: str, + *, + label: str, + not_found_message: str, +) -> Tuple[Dict[str, Any], int]: + """Reset a discovery playlist back to the 'fresh' phase, clearing all + discovery/sync data while preserving the original playlist payload. + + 1:1 lift of the byte-identical ``reset__playlist`` bodies + (Tidal, Deezer, Qobuz, Spotify-Public). Returns ``(payload, status_code)``. + + NOT folded in (genuinely divergent): YouTube (status -> 'parsed', no + download_process_id, logs the playlist name, "reset to fresh state"), + ListenBrainz (status -> 'cached', logs playlist title, returns + {"phase": "fresh"}), iTunes-Link (uses state.update, no info log, distinct + message). Those keep their own bodies. + """ + try: + if key not in states: + return {"error": not_found_message}, 404 + + state = states[key] + if 'discovery_future' in state and state['discovery_future']: + state['discovery_future'].cancel() + + state['phase'] = 'fresh' + state['status'] = 'fresh' + state['discovery_results'] = [] + state['discovery_progress'] = 0 + state['spotify_matches'] = 0 + state['sync_playlist_id'] = None + state['converted_spotify_playlist_id'] = None + state['download_process_id'] = None + state['sync_progress'] = {} + state['discovery_future'] = None + state['last_accessed'] = time.time() + + logger.info(f"Reset {label} playlist to fresh: {key}") + return {"success": True, "message": "Playlist reset to fresh phase"}, 200 + except Exception as e: + logger.error(f"Error resetting {label} playlist: {e}") + return {"error": str(e)}, 500 diff --git a/tests/discovery/test_discovery_endpoints.py b/tests/discovery/test_discovery_endpoints.py index 846802f5..03cd31e5 100644 --- a/tests/discovery/test_discovery_endpoints.py +++ b/tests/discovery/test_discovery_endpoints.py @@ -445,3 +445,63 @@ def test_discovery_status_missing_field_raises_500(): body, code = get_discovery_status(states, 'pl', not_found_message='nf', error_label='Beatport') assert code == 500 and "error" in body + + +# --------------------------------------------------------------------------- +# reset_playlist +# --------------------------------------------------------------------------- + +def test_reset_not_found(): + from core.discovery.endpoints import reset_playlist + body, code = reset_playlist({}, 'missing', label='Tidal', + not_found_message='Tidal playlist not found') + assert code == 404 and body == {"error": "Tidal playlist not found"} + + +def test_reset_clears_state_preserving_playlist_and_cancels_future(): + from core.discovery.endpoints import reset_playlist + fut = _FakeFuture() + state = { + 'playlist': {'name': 'keep me'}, + 'phase': 'discovered', 'status': 'done', + 'discovery_results': [{'x': 1}], 'discovery_progress': 100, + 'spotify_matches': 5, 'sync_playlist_id': 'sp', 'last_accessed': 0, + 'converted_spotify_playlist_id': 'cv', 'download_process_id': 'dp', + 'sync_progress': {'n': 1}, 'discovery_future': fut, + } + states = {'pl': state} + body, code = reset_playlist(states, 'pl', label='Tidal', not_found_message='nf') + + assert code == 200 + assert body == {"success": True, "message": "Playlist reset to fresh phase"} + assert fut.cancelled is True + # cleared + assert state['phase'] == 'fresh' + assert state['status'] == 'fresh' + assert state['discovery_results'] == [] + assert state['discovery_progress'] == 0 + assert state['spotify_matches'] == 0 + assert state['sync_playlist_id'] is None + assert state['converted_spotify_playlist_id'] is None + assert state['download_process_id'] is None + assert state['sync_progress'] == {} + assert state['discovery_future'] is None + assert state['last_accessed'] != 0 + # original playlist payload preserved + assert state['playlist'] == {'name': 'keep me'} + + +def test_reset_without_discovery_future(): + from core.discovery.endpoints import reset_playlist + state = {'phase': 'discovered'} # no discovery_future key + body, code = reset_playlist({'pl': state}, 'pl', label='Deezer', not_found_message='nf') + assert code == 200 + assert state['phase'] == 'fresh' + + +def test_reset_exception_returns_500(): + from core.discovery.endpoints import reset_playlist + fut = object() # .cancel missing -> AttributeError in try + states = {'pl': {'discovery_future': fut}} + body, code = reset_playlist(states, 'pl', label='Qobuz', not_found_message='nf') + assert code == 500 and "error" in body diff --git a/web_server.py b/web_server.py index 9ad29b8c..ccde67a1 100644 --- a/web_server.py +++ b/web_server.py @@ -20748,35 +20748,7 @@ def get_tidal_playlist_state(playlist_id): @app.route('/api/tidal/reset/', methods=['POST']) def reset_tidal_playlist(playlist_id): """Reset Tidal playlist to fresh phase (clear discovery/sync data)""" - try: - if playlist_id not in tidal_discovery_states: - return jsonify({"error": "Tidal playlist not found"}), 404 - - state = tidal_discovery_states[playlist_id] - - # Stop any active discovery - if 'discovery_future' in state and state['discovery_future']: - state['discovery_future'].cancel() - - # Reset state to fresh (preserve original playlist data) - state['phase'] = 'fresh' - state['status'] = 'fresh' - state['discovery_results'] = [] - state['discovery_progress'] = 0 - state['spotify_matches'] = 0 - state['sync_playlist_id'] = None - state['converted_spotify_playlist_id'] = None - state['download_process_id'] = None - state['sync_progress'] = {} - state['discovery_future'] = None - state['last_accessed'] = time.time() - - logger.info(f"Reset Tidal playlist to fresh: {playlist_id}") - return jsonify({"success": True, "message": "Playlist reset to fresh phase"}) - - except Exception as e: - logger.error(f"Error resetting Tidal playlist: {e}") - return jsonify({"error": str(e)}), 500 + return _reset_source_playlist(tidal_discovery_states, playlist_id, "Tidal", "Tidal playlist not found") @app.route('/api/tidal/delete/', methods=['POST']) def delete_tidal_playlist(playlist_id): @@ -21041,6 +21013,7 @@ from core.discovery.endpoints import ( delete_playlist_state as _delete_playlist_state_core, get_sync_status as _get_sync_status_core, get_discovery_status as _get_discovery_status_core, + reset_playlist as _reset_playlist_core, playlist_name_attr_or_unknown as _pl_name_attr_or_unknown, playlist_name_strict as _pl_name_strict, playlist_name_safe as _pl_name_safe, @@ -21087,6 +21060,15 @@ def _get_source_discovery_status(states, key, not_found_message, error_label): return jsonify(body), code +def _reset_source_playlist(states, key, label, not_found_message): + """Thin glue for the per-source reset routes that share the identical + reset body (Tidal/Deezer/Qobuz/Spotify-Public).""" + body, code = _reset_playlist_core( + states, key, label=label, not_found_message=not_found_message, + ) + return jsonify(body), code + + def _build_tidal_discovery_deps(): """Build the TidalDiscoveryDeps bundle from web_server.py globals on each call.""" return _discovery_tidal.TidalDiscoveryDeps( @@ -21588,35 +21570,7 @@ def get_deezer_playlist_state(playlist_id): @app.route('/api/deezer/reset/', methods=['POST']) def reset_deezer_playlist(playlist_id): """Reset Deezer playlist to fresh phase (clear discovery/sync data)""" - try: - if playlist_id not in deezer_discovery_states: - return jsonify({"error": "Deezer playlist not found"}), 404 - - state = deezer_discovery_states[playlist_id] - - # Stop any active discovery - if 'discovery_future' in state and state['discovery_future']: - state['discovery_future'].cancel() - - # Reset state to fresh (preserve original playlist data) - state['phase'] = 'fresh' - state['status'] = 'fresh' - state['discovery_results'] = [] - state['discovery_progress'] = 0 - state['spotify_matches'] = 0 - state['sync_playlist_id'] = None - state['converted_spotify_playlist_id'] = None - state['download_process_id'] = None - state['sync_progress'] = {} - state['discovery_future'] = None - state['last_accessed'] = time.time() - - logger.info(f"Reset Deezer playlist to fresh: {playlist_id}") - return jsonify({"success": True, "message": "Playlist reset to fresh phase"}) - - except Exception as e: - logger.error(f"Error resetting Deezer playlist: {e}") - return jsonify({"error": str(e)}), 500 + return _reset_source_playlist(deezer_discovery_states, playlist_id, "Deezer", "Deezer playlist not found") @app.route('/api/deezer/delete/', methods=['POST']) def delete_deezer_playlist(playlist_id): @@ -22122,33 +22076,7 @@ def get_qobuz_playlist_state(playlist_id): @app.route('/api/qobuz/reset/', methods=['POST']) def reset_qobuz_playlist(playlist_id): """Reset Qobuz playlist to fresh phase (clear discovery/sync data).""" - try: - if playlist_id not in qobuz_discovery_states: - return jsonify({"error": "Qobuz playlist not found"}), 404 - - state = qobuz_discovery_states[playlist_id] - - if 'discovery_future' in state and state['discovery_future']: - state['discovery_future'].cancel() - - state['phase'] = 'fresh' - state['status'] = 'fresh' - state['discovery_results'] = [] - state['discovery_progress'] = 0 - state['spotify_matches'] = 0 - state['sync_playlist_id'] = None - state['converted_spotify_playlist_id'] = None - state['download_process_id'] = None - state['sync_progress'] = {} - state['discovery_future'] = None - state['last_accessed'] = time.time() - - logger.info(f"Reset Qobuz playlist to fresh: {playlist_id}") - return jsonify({"success": True, "message": "Playlist reset to fresh phase"}) - - except Exception as e: - logger.error(f"Error resetting Qobuz playlist: {e}") - return jsonify({"error": str(e)}), 500 + return _reset_source_playlist(qobuz_discovery_states, playlist_id, "Qobuz", "Qobuz playlist not found") @app.route('/api/qobuz/delete/', methods=['POST']) @@ -22930,35 +22858,7 @@ def get_spotify_public_playlist_state(url_hash): @app.route('/api/spotify-public/reset/', methods=['POST']) def reset_spotify_public_playlist(url_hash): """Reset Spotify Public playlist to fresh phase (clear discovery/sync data)""" - try: - if url_hash not in spotify_public_discovery_states: - return jsonify({"error": "Spotify Public playlist not found"}), 404 - - state = spotify_public_discovery_states[url_hash] - - # Stop any active discovery - if 'discovery_future' in state and state['discovery_future']: - state['discovery_future'].cancel() - - # Reset state to fresh (preserve original playlist data) - state['phase'] = 'fresh' - state['status'] = 'fresh' - state['discovery_results'] = [] - state['discovery_progress'] = 0 - state['spotify_matches'] = 0 - state['sync_playlist_id'] = None - state['converted_spotify_playlist_id'] = None - state['download_process_id'] = None - state['sync_progress'] = {} - state['discovery_future'] = None - state['last_accessed'] = time.time() - - logger.info(f"Reset Spotify Public playlist to fresh: {url_hash}") - return jsonify({"success": True, "message": "Playlist reset to fresh phase"}) - - except Exception as e: - logger.error(f"Error resetting Spotify Public playlist: {e}") - return jsonify({"error": str(e)}), 500 + return _reset_source_playlist(spotify_public_discovery_states, url_hash, "Spotify Public", "Spotify Public playlist not found") @app.route('/api/spotify-public/delete/', methods=['POST']) def delete_spotify_public_playlist(url_hash): From 7b6615b65adb6f2e09b7214e1df9a088c0b943dc Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 17:31:33 -0700 Subject: [PATCH 15/52] Discovery lift (6/N): get_*_playlist_states -> shared helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sixth cluster: the bulk-hydration get__playlist_states endpoints for the five sources that build the identical per-entry dict + {"states": [...]} shape (Tidal, Deezer, Qobuz, Spotify-Public, iTunes-Link) -> core.discovery.endpoints.get_playlist_states(states, *, error_label, info_log_label=None), wired via _get_source_playlist_states. iTunes-Link is the only one of the five without the "Returning N stored ..." info log, so info_log_label is optional (iTunes passes None to suppress it). NOT folded in: the YouTube/ListenBrainz get_all_*_playlists endpoints. They return {"playlists": [...]} (different key) with a different field set (url / created_at / playlist, no discovery_results) and filter out mirrored_/profile-scoped entries — genuinely divergent, kept as-is. Tests: +4 (list build + last_accessed bump + exact shape, empty, optional ids default None, missing-required-field -> 500). Full discovery suite: 183 passed. web_server.py: -116 lines. --- core/discovery/endpoints.py | 45 ++++++ tests/discovery/test_discovery_endpoints.py | 44 ++++++ web_server.py | 146 ++------------------ 3 files changed, 104 insertions(+), 131 deletions(-) diff --git a/core/discovery/endpoints.py b/core/discovery/endpoints.py index 038dd889..b86f4f1e 100644 --- a/core/discovery/endpoints.py +++ b/core/discovery/endpoints.py @@ -334,3 +334,48 @@ def reset_playlist( except Exception as e: logger.error(f"Error resetting {label} playlist: {e}") return {"error": str(e)}, 500 + + +def get_playlist_states( + states: Dict[str, Any], + *, + error_label: str, + info_log_label: str = None, +) -> Tuple[Dict[str, Any], int]: + """Return all stored discovery states for a source as a list for frontend + card hydration (``{"states": [...]}``). + + 1:1 lift of the ``get__playlist_states`` bodies (Tidal, Deezer, + Qobuz, Spotify-Public, iTunes-Link), which build the same per-entry dict. + iTunes-Link is the only one without the "Returning N ..." info log, so + ``info_log_label`` is optional (pass None to suppress it, as iTunes did). + + NOT folded in: the YouTube/ListenBrainz ``get_all_*_playlists`` endpoints — + they return ``{"playlists": [...]}`` (different key + fields: url/created_at, + no discovery_results) and filter mirrored/profile-scoped entries. + """ + try: + result = [] + current_time = time.time() + + for key, state in states.items(): + state['last_accessed'] = current_time + result.append({ + 'playlist_id': key, + 'phase': state['phase'], + 'status': state['status'], + 'discovery_progress': state['discovery_progress'], + 'spotify_matches': state['spotify_matches'], + 'spotify_total': state['spotify_total'], + 'discovery_results': state['discovery_results'], + 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), + 'download_process_id': state.get('download_process_id'), + 'last_accessed': state['last_accessed'], + }) + + if info_log_label: + logger.info(f"Returning {len(result)} stored {info_log_label} playlist states for hydration") + return {"states": result}, 200 + except Exception as e: + logger.error(f"Error getting {error_label} playlist states: {e}") + return {"error": str(e)}, 500 diff --git a/tests/discovery/test_discovery_endpoints.py b/tests/discovery/test_discovery_endpoints.py index 03cd31e5..0d0d5e5d 100644 --- a/tests/discovery/test_discovery_endpoints.py +++ b/tests/discovery/test_discovery_endpoints.py @@ -505,3 +505,47 @@ def test_reset_exception_returns_500(): states = {'pl': {'discovery_future': fut}} body, code = reset_playlist(states, 'pl', label='Qobuz', not_found_message='nf') assert code == 500 and "error" in body + + +# --------------------------------------------------------------------------- +# get_playlist_states (bulk hydration list) +# --------------------------------------------------------------------------- + +def test_playlist_states_builds_list_and_bumps_access(): + from core.discovery.endpoints import get_playlist_states + s1 = {'phase': 'discovered', 'status': 'done', 'discovery_progress': 100, + 'spotify_matches': 3, 'spotify_total': 5, 'discovery_results': [{'a': 1}], + 'converted_spotify_playlist_id': 'cv', 'download_process_id': 'dp', + 'last_accessed': 0} + states = {'k1': s1} + body, code = get_playlist_states(states, error_label='Tidal', info_log_label='Tidal') + assert code == 200 + assert body == {"states": [{ + 'playlist_id': 'k1', 'phase': 'discovered', 'status': 'done', + 'discovery_progress': 100, 'spotify_matches': 3, 'spotify_total': 5, + 'discovery_results': [{'a': 1}], 'converted_spotify_playlist_id': 'cv', + 'download_process_id': 'dp', 'last_accessed': s1['last_accessed'], + }]} + assert s1['last_accessed'] != 0 # bumped + + +def test_playlist_states_empty(): + from core.discovery.endpoints import get_playlist_states + body, code = get_playlist_states({}, error_label='Deezer', info_log_label='Deezer') + assert code == 200 and body == {"states": []} + + +def test_playlist_states_optional_ids_default_none(): + from core.discovery.endpoints import get_playlist_states + state = {'phase': 'fresh', 'status': 'fresh', 'discovery_progress': 0, + 'spotify_matches': 0, 'spotify_total': 0, 'discovery_results': []} + body, _ = get_playlist_states({'k': state}, error_label='iTunes Link') + assert body['states'][0]['converted_spotify_playlist_id'] is None + assert body['states'][0]['download_process_id'] is None + + +def test_playlist_states_missing_required_field_raises_500(): + from core.discovery.endpoints import get_playlist_states + # state missing 'phase' -> strict access raises -> 500 + body, code = get_playlist_states({'k': {'status': 'x'}}, error_label='Qobuz') + assert code == 500 and "error" in body diff --git a/web_server.py b/web_server.py index ccde67a1..b3dcbe71 100644 --- a/web_server.py +++ b/web_server.py @@ -20682,35 +20682,7 @@ def update_tidal_discovery_match(): @app.route('/api/tidal/playlists/states', methods=['GET']) def get_tidal_playlist_states(): """Get all stored Tidal playlist discovery states for frontend hydration (similar to YouTube playlists)""" - try: - states = [] - current_time = time.time() - - for playlist_id, state in tidal_discovery_states.items(): - # Update access time when requested - state['last_accessed'] = current_time - - # Return essential data for card state recreation - state_info = { - 'playlist_id': playlist_id, - 'phase': state['phase'], - 'status': state['status'], - 'discovery_progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'discovery_results': state['discovery_results'], - 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), - 'download_process_id': state.get('download_process_id'), - 'last_accessed': state['last_accessed'] - } - states.append(state_info) - - logger.info(f"Returning {len(states)} stored Tidal playlist states for hydration") - return jsonify({"states": states}) - - except Exception as e: - logger.error(f"Error getting Tidal playlist states: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_playlist_states(tidal_discovery_states, "Tidal", "Tidal") @app.route('/api/tidal/state/', methods=['GET']) def get_tidal_playlist_state(playlist_id): @@ -21014,6 +20986,7 @@ from core.discovery.endpoints import ( get_sync_status as _get_sync_status_core, get_discovery_status as _get_discovery_status_core, reset_playlist as _reset_playlist_core, + get_playlist_states as _get_playlist_states_core, playlist_name_attr_or_unknown as _pl_name_attr_or_unknown, playlist_name_strict as _pl_name_strict, playlist_name_safe as _pl_name_safe, @@ -21069,6 +21042,15 @@ def _reset_source_playlist(states, key, label, not_found_message): return jsonify(body), code +def _get_source_playlist_states(states, error_label, info_log_label=None): + """Thin glue for the per-source get_*_playlist_states bulk-hydration routes + (Tidal/Deezer/Qobuz/Spotify-Public/iTunes-Link).""" + body, code = _get_playlist_states_core( + states, error_label=error_label, info_log_label=info_log_label, + ) + return jsonify(body), code + + def _build_tidal_discovery_deps(): """Build the TidalDiscoveryDeps bundle from web_server.py globals on each call.""" return _discovery_tidal.TidalDiscoveryDeps( @@ -21506,33 +21488,7 @@ def update_deezer_discovery_match(): @app.route('/api/deezer/playlists/states', methods=['GET']) def get_deezer_playlist_states(): """Get all stored Deezer playlist discovery states for frontend hydration""" - try: - states = [] - current_time = time.time() - - for playlist_id, state in deezer_discovery_states.items(): - state['last_accessed'] = current_time - - state_info = { - 'playlist_id': playlist_id, - 'phase': state['phase'], - 'status': state['status'], - 'discovery_progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'discovery_results': state['discovery_results'], - 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), - 'download_process_id': state.get('download_process_id'), - 'last_accessed': state['last_accessed'] - } - states.append(state_info) - - logger.info(f"Returning {len(states)} stored Deezer playlist states for hydration") - return jsonify({"states": states}) - - except Exception as e: - logger.error(f"Error getting Deezer playlist states: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_playlist_states(deezer_discovery_states, "Deezer", "Deezer") @app.route('/api/deezer/state/', methods=['GET']) def get_deezer_playlist_state(playlist_id): @@ -22011,33 +21967,7 @@ def update_qobuz_discovery_match(): @app.route('/api/qobuz/playlists/states', methods=['GET']) def get_qobuz_playlist_states(): """Get all stored Qobuz playlist discovery states for frontend hydration.""" - try: - states = [] - current_time = time.time() - - for playlist_id, state in qobuz_discovery_states.items(): - state['last_accessed'] = current_time - - state_info = { - 'playlist_id': playlist_id, - 'phase': state['phase'], - 'status': state['status'], - 'discovery_progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'discovery_results': state['discovery_results'], - 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), - 'download_process_id': state.get('download_process_id'), - 'last_accessed': state['last_accessed'] - } - states.append(state_info) - - logger.info(f"Returning {len(states)} stored Qobuz playlist states for hydration") - return jsonify({"states": states}) - - except Exception as e: - logger.error(f"Error getting Qobuz playlist states: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_playlist_states(qobuz_discovery_states, "Qobuz", "Qobuz") @app.route('/api/qobuz/state/', methods=['GET']) @@ -22795,33 +22725,7 @@ def update_spotify_public_discovery_match(): @app.route('/api/spotify-public/playlists/states', methods=['GET']) def get_spotify_public_playlist_states(): """Get all stored Spotify Public playlist discovery states for frontend hydration""" - try: - states = [] - current_time = time.time() - - for url_hash, state in spotify_public_discovery_states.items(): - state['last_accessed'] = current_time - - state_info = { - 'playlist_id': url_hash, - 'phase': state['phase'], - 'status': state['status'], - 'discovery_progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'discovery_results': state['discovery_results'], - 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), - 'download_process_id': state.get('download_process_id'), - 'last_accessed': state['last_accessed'] - } - states.append(state_info) - - logger.info(f"Returning {len(states)} stored Spotify Public playlist states for hydration") - return jsonify({"states": states}) - - except Exception as e: - logger.error(f"Error getting Spotify Public playlist states: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_playlist_states(spotify_public_discovery_states, "Spotify Public", "Spotify Public") @app.route('/api/spotify-public/state/', methods=['GET']) def get_spotify_public_playlist_state(url_hash): @@ -23205,27 +23109,7 @@ def update_itunes_link_discovery_match(): @app.route('/api/itunes-link/playlists/states', methods=['GET']) def get_itunes_link_playlist_states(): - try: - current_time = time.time() - states = [] - for url_hash, state in itunes_link_discovery_states.items(): - state['last_accessed'] = current_time - states.append({ - 'playlist_id': url_hash, - 'phase': state['phase'], - 'status': state['status'], - 'discovery_progress': state['discovery_progress'], - 'spotify_matches': state['spotify_matches'], - 'spotify_total': state['spotify_total'], - 'discovery_results': state['discovery_results'], - 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), - 'download_process_id': state.get('download_process_id'), - 'last_accessed': state['last_accessed'] - }) - return jsonify({"states": states}) - except Exception as e: - logger.error(f"Error getting iTunes Link playlist states: {e}") - return jsonify({"error": str(e)}), 500 + return _get_source_playlist_states(itunes_link_discovery_states, "iTunes Link") @app.route('/api/itunes-link/state/', methods=['GET']) From 17c9e9b7b9773f94a540c20201cdb3ad28a6a43b Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 17:45:09 -0700 Subject: [PATCH 16/52] Discovery lift (7/N): start_*_sync -> shared helper Seventh cluster: start__sync for the five sources with the identical flow (Tidal, Deezer, Qobuz, Spotify-Public, YouTube) -> core.discovery.endpoints.start_sync(...), wired via _start_source_sync. Validates phase, converts discovery results, seeds sync state, posts a "... Sync Started" activity item, and submits to the sync executor. Per-source pieces are params: - sync_id_prefix (f"{prefix}_{key}"), not_found/not_ready messages, convert_fn. - name/image accessors: Tidal reads an object (playlist_name_obj/ playlist_image_obj), the rest a dict (playlist_name_strict/playlist_image_dict). - activity_label vs error_label DIFFER for Spotify-Public ("Spotify Link Sync Started" activity, "Spotify Public" logs). - submit_sync_task glue (_submit_sync_task) closes over sync_executor / _run_sync_task / get_current_profile_id so the helper stays global-free. NOT folded in: iTunes-Link (no final info log), ListenBrainz (submits the task WITHOUT a playlist_image_url arg), Beatport (extra debug logging, chart). Tests: +6 (404, not-ready 400, no-matches 400, full happy path with state/sync-infra/submit/activity assertions, resync phases allowed, exception -> 500). Full discovery suite: 189 passed. web_server.py: -172 lines. --- core/discovery/endpoints.py | 95 ++++++ tests/discovery/test_discovery_endpoints.py | 103 +++++++ web_server.py | 304 +++++--------------- 3 files changed, 264 insertions(+), 238 deletions(-) diff --git a/core/discovery/endpoints.py b/core/discovery/endpoints.py index b86f4f1e..32fe809b 100644 --- a/core/discovery/endpoints.py +++ b/core/discovery/endpoints.py @@ -182,6 +182,23 @@ def playlist_name_safe(state: Dict[str, Any]) -> str: return state.get('playlist', {}).get('name', 'Unknown Playlist') +def playlist_name_obj(state: Dict[str, Any]) -> str: + """Tidal start-sync: playlist is an object — strict ``.name`` (raises if + absent, exactly like the original).""" + return state['playlist'].name + + +def playlist_image_obj(state: Dict[str, Any]) -> str: + """Tidal: ``getattr(playlist, 'image_url', '')`` (object attribute).""" + return getattr(state['playlist'], 'image_url', '') + + +def playlist_image_dict(state: Dict[str, Any]) -> str: + """Deezer/Qobuz/Spotify-Public/YouTube: ``playlist.get('image_url', '')`` + (dict access).""" + return state['playlist'].get('image_url', '') + + def get_sync_status( states: Dict[str, Any], key: str, @@ -379,3 +396,81 @@ def get_playlist_states( except Exception as e: logger.error(f"Error getting {error_label} playlist states: {e}") return {"error": str(e)}, 500 + + +def start_sync( + states: Dict[str, Any], + key: str, + *, + sync_id_prefix: str, + not_found_message: str, + not_ready_message: str, + convert_fn, + playlist_name_getter, + playlist_image_getter, + activity_label: str, + error_label: str, + sync_lock: Any, + sync_states: Dict[str, Any], + active_sync_workers: Dict[str, Any], + submit_sync_task, + add_activity_item, +) -> Tuple[Dict[str, Any], int]: + """Kick off a playlist sync from a source's discovered Spotify matches. + + 1:1 lift of the ``start__sync`` bodies for the five sources with + the identical flow (Tidal, Deezer, Qobuz, Spotify-Public, YouTube). The + per-source pieces are parameters: + + - ``sync_id_prefix`` — the ``f"{prefix}_{key}"`` sync id. + - ``convert_fn`` — the source's discovery->spotify-tracks converter. + - ``playlist_name_getter`` / ``playlist_image_getter`` — Tidal reads an + object (``.name`` / ``getattr``), the rest read a dict; lifted as the + ``playlist_name_obj``/``playlist_image_obj`` vs ``playlist_name_strict``/ + ``playlist_image_dict`` accessors. + - ``activity_label`` vs ``error_label`` — these DIFFER for Spotify-Public: + activity says "Spotify Link Sync Started" while logs say "Spotify Public". + - ``submit_sync_task(sync_playlist_id, playlist_name, spotify_tracks, + playlist_image_url) -> Future`` — wraps sync_executor/_run_sync_task/ + get_current_profile_id so this stays free of those globals. + + Returns ``(payload, status_code)``. + + NOT folded in: iTunes-Link (no final info log), ListenBrainz (submits the + task without an image arg), Beatport (extra debug logging, 'chart' key). + """ + try: + if key not in states: + return {"error": not_found_message}, 404 + + state = states[key] + state['last_accessed'] = time.time() + + if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: + return {"error": not_ready_message}, 400 + + spotify_tracks = convert_fn(state['discovery_results']) + if not spotify_tracks: + return {"error": "No Spotify matches found for sync"}, 400 + + sync_playlist_id = f"{sync_id_prefix}_{key}" + playlist_name = playlist_name_getter(state) + + add_activity_item("", f"{activity_label} Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") + + state['phase'] = 'syncing' + state['sync_playlist_id'] = sync_playlist_id + state['sync_progress'] = {} + + with sync_lock: + sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} + + playlist_image_url = playlist_image_getter(state) + future = submit_sync_task(sync_playlist_id, playlist_name, spotify_tracks, playlist_image_url) + active_sync_workers[sync_playlist_id] = future + + logger.info(f"Started {error_label} sync for: {playlist_name} ({len(spotify_tracks)} tracks)") + return {"success": True, "sync_playlist_id": sync_playlist_id}, 200 + except Exception as e: + logger.error(f"Error starting {error_label} sync: {e}") + return {"error": str(e)}, 500 diff --git a/tests/discovery/test_discovery_endpoints.py b/tests/discovery/test_discovery_endpoints.py index 0d0d5e5d..219dfd77 100644 --- a/tests/discovery/test_discovery_endpoints.py +++ b/tests/discovery/test_discovery_endpoints.py @@ -549,3 +549,106 @@ def test_playlist_states_missing_required_field_raises_500(): # state missing 'phase' -> strict access raises -> 500 body, code = get_playlist_states({'k': {'status': 'x'}}, error_label='Qobuz') assert code == 500 and "error" in body + + +# --------------------------------------------------------------------------- +# start_sync +# --------------------------------------------------------------------------- + +def _start_kwargs(infra, *, convert_fn=None, name='PL', image='img', + activity_label='Tidal', error_label='Tidal', + not_found_message='Tidal playlist not found', + not_ready_message='Tidal playlist not ready for sync', + sync_id_prefix='tidal'): + submitted = [] + + def submit(sync_playlist_id, playlist_name, tracks, image_url): + rec = (sync_playlist_id, playlist_name, tracks, image_url) + submitted.append(rec) + return f"future:{sync_playlist_id}" + + calls, add = _activity_recorder() + kw = dict( + sync_id_prefix=sync_id_prefix, not_found_message=not_found_message, + not_ready_message=not_ready_message, + convert_fn=convert_fn or (lambda r: [{'id': 't'}]), + playlist_name_getter=lambda s: name, playlist_image_getter=lambda s: image, + activity_label=activity_label, error_label=error_label, + sync_lock=infra['sync_lock'], sync_states=infra['sync_states'], + active_sync_workers=infra['active_sync_workers'], + submit_sync_task=submit, add_activity_item=add, + ) + return kw, submitted, calls + + +def test_start_sync_not_found(): + from core.discovery.endpoints import start_sync + infra = _cancel_infra() + kw, _, _ = _start_kwargs(infra) + body, code = start_sync({}, 'missing', **kw) + assert code == 404 and body == {"error": "Tidal playlist not found"} + + +def test_start_sync_not_ready_phase(): + from core.discovery.endpoints import start_sync + infra = _cancel_infra() + kw, _, _ = _start_kwargs(infra) + states = {'pl': {'phase': 'discovering'}} + body, code = start_sync(states, 'pl', **kw) + assert code == 400 and body == {"error": "Tidal playlist not ready for sync"} + + +def test_start_sync_no_matches(): + from core.discovery.endpoints import start_sync + infra = _cancel_infra() + kw, _, _ = _start_kwargs(infra, convert_fn=lambda r: []) + states = {'pl': {'phase': 'discovered', 'discovery_results': []}} + body, code = start_sync(states, 'pl', **kw) + assert code == 400 and body == {"error": "No Spotify matches found for sync"} + + +def test_start_sync_happy_path(): + from core.discovery.endpoints import start_sync + infra = _cancel_infra() + kw, submitted, calls = _start_kwargs( + infra, convert_fn=lambda r: [{'id': 'a'}, {'id': 'b'}], + name='My Mix', image='cover.jpg', + activity_label='Spotify Link', error_label='Spotify Public', + sync_id_prefix='spotify_public') + states = {'h1': {'phase': 'discovered', 'discovery_results': [1, 2]}} + body, code = start_sync(states, 'h1', **kw) + + assert code == 200 + assert body == {"success": True, "sync_playlist_id": "spotify_public_h1"} + # state mutated + assert states['h1']['phase'] == 'syncing' + assert states['h1']['sync_playlist_id'] == 'spotify_public_h1' + assert states['h1']['sync_progress'] == {} + # sync infra seeded + worker registered + assert infra['sync_states']['spotify_public_h1'] == {"status": "starting", "progress": {}} + assert infra['active_sync_workers']['spotify_public_h1'] == 'future:spotify_public_h1' + # submit got name/tracks/image + assert submitted == [('spotify_public_h1', 'My Mix', [{'id': 'a'}, {'id': 'b'}], 'cover.jpg')] + # activity uses activity_label (not error_label) + track count + assert calls == [("", "Spotify Link Sync Started", "'My Mix' - 2 tracks", "Now")] + + +def test_start_sync_allows_resync_phases(): + from core.discovery.endpoints import start_sync + for phase in ('sync_complete', 'download_complete'): + infra = _cancel_infra() + kw, _, _ = _start_kwargs(infra) + states = {'pl': {'phase': phase, 'discovery_results': [1]}} + body, code = start_sync(states, 'pl', **kw) + assert code == 200, phase + + +def test_start_sync_exception_returns_500(): + from core.discovery.endpoints import start_sync + infra = _cancel_infra() + def boom(r): + raise RuntimeError("convert blew up") + kw, _, _ = _start_kwargs(infra, convert_fn=boom) + states = {'pl': {'phase': 'discovered', 'discovery_results': [1]}} + body, code = start_sync(states, 'pl', **kw) + assert code == 500 and "error" in body diff --git a/web_server.py b/web_server.py index b3dcbe71..7540989a 100644 --- a/web_server.py +++ b/web_server.py @@ -20987,9 +20987,13 @@ from core.discovery.endpoints import ( get_discovery_status as _get_discovery_status_core, reset_playlist as _reset_playlist_core, get_playlist_states as _get_playlist_states_core, + start_sync as _start_sync_core, playlist_name_attr_or_unknown as _pl_name_attr_or_unknown, playlist_name_strict as _pl_name_strict, playlist_name_safe as _pl_name_safe, + playlist_name_obj as _pl_name_obj, + playlist_image_obj as _pl_image_obj, + playlist_image_dict as _pl_image_dict, ) @@ -21051,6 +21055,33 @@ def _get_source_playlist_states(states, error_label, info_log_label=None): return jsonify(body), code +def _submit_sync_task(sync_playlist_id, playlist_name, spotify_tracks, playlist_image_url): + """Submit a sync to the shared executor (closes over sync_executor / + _run_sync_task / get_current_profile_id so the lifted start_sync helper + stays free of those globals).""" + return sync_executor.submit( + _run_sync_task, sync_playlist_id, playlist_name, spotify_tracks, + None, get_current_profile_id(), playlist_image_url, + ) + + +def _start_source_sync(states, key, *, sync_id_prefix, not_found_message, + not_ready_message, convert_fn, name_getter, image_getter, + activity_label, error_label): + """Thin glue for the per-source start_*_sync routes (Tidal/Deezer/Qobuz/ + Spotify-Public/YouTube) — wires sync infra into the lifted start_sync.""" + body, code = _start_sync_core( + states, key, sync_id_prefix=sync_id_prefix, + not_found_message=not_found_message, not_ready_message=not_ready_message, + convert_fn=convert_fn, playlist_name_getter=name_getter, + playlist_image_getter=image_getter, activity_label=activity_label, + error_label=error_label, sync_lock=sync_lock, sync_states=sync_states, + active_sync_workers=active_sync_workers, submit_sync_task=_submit_sync_task, + add_activity_item=add_activity_item, + ) + return jsonify(body), code + + def _build_tidal_discovery_deps(): """Build the TidalDiscoveryDeps bundle from web_server.py globals on each call.""" return _discovery_tidal.TidalDiscoveryDeps( @@ -21089,55 +21120,13 @@ def convert_tidal_results_to_spotify_tracks(discovery_results): @app.route('/api/tidal/sync/start/', methods=['POST']) def start_tidal_sync(playlist_id): """Start sync process for a Tidal playlist using discovered Spotify tracks""" - try: - if playlist_id not in tidal_discovery_states: - return jsonify({"error": "Tidal playlist not found"}), 404 - - state = tidal_discovery_states[playlist_id] - state['last_accessed'] = time.time() # Update access time - - if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: - return jsonify({"error": "Tidal playlist not ready for sync"}), 400 - - # Convert discovery results to Spotify tracks format - spotify_tracks = convert_tidal_results_to_spotify_tracks(state['discovery_results']) - - if not spotify_tracks: - return jsonify({"error": "No Spotify matches found for sync"}), 400 - - # Create a temporary playlist ID for sync tracking - sync_playlist_id = f"tidal_{playlist_id}" - playlist_name = state['playlist'].name # Tidal playlist object has .name attribute - - # Add activity for sync start - add_activity_item("", "Tidal Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") - - # Update Tidal state - state['phase'] = 'syncing' - state['sync_playlist_id'] = sync_playlist_id - state['sync_progress'] = {} - - # Start the sync using existing sync infrastructure - sync_data = { - 'playlist_id': sync_playlist_id, - 'playlist_name': playlist_name, - 'tracks': spotify_tracks - } - - with sync_lock: - sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} - - # Submit sync task - playlist_image_url = getattr(state['playlist'], 'image_url', '') - future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks, None, get_current_profile_id(), playlist_image_url) - active_sync_workers[sync_playlist_id] = future - - logger.info(f"Started Tidal sync for: {playlist_name} ({len(spotify_tracks)} tracks)") - return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) - - except Exception as e: - logger.error(f"Error starting Tidal sync: {e}") - return jsonify({"error": str(e)}), 500 + return _start_source_sync( + tidal_discovery_states, playlist_id, sync_id_prefix="tidal", + not_found_message="Tidal playlist not found", + not_ready_message="Tidal playlist not ready for sync", + convert_fn=convert_tidal_results_to_spotify_tracks, + name_getter=_pl_name_obj, image_getter=_pl_image_obj, + activity_label="Tidal", error_label="Tidal") @app.route('/api/tidal/sync/status/', methods=['GET']) def get_tidal_sync_status(playlist_id): @@ -21611,55 +21600,13 @@ def convert_deezer_results_to_spotify_tracks(discovery_results): @app.route('/api/deezer/sync/start/', methods=['POST']) def start_deezer_sync(playlist_id): """Start sync process for a Deezer playlist using discovered Spotify tracks""" - try: - if playlist_id not in deezer_discovery_states: - return jsonify({"error": "Deezer playlist not found"}), 404 - - state = deezer_discovery_states[playlist_id] - state['last_accessed'] = time.time() - - if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: - return jsonify({"error": "Deezer playlist not ready for sync"}), 400 - - # Convert discovery results to Spotify tracks format - spotify_tracks = convert_deezer_results_to_spotify_tracks(state['discovery_results']) - - if not spotify_tracks: - return jsonify({"error": "No Spotify matches found for sync"}), 400 - - # Create a temporary playlist ID for sync tracking - sync_playlist_id = f"deezer_{playlist_id}" - playlist_name = state['playlist']['name'] - - # Add activity for sync start - add_activity_item("", "Deezer Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") - - # Update Deezer state - state['phase'] = 'syncing' - state['sync_playlist_id'] = sync_playlist_id - state['sync_progress'] = {} - - # Start the sync using existing sync infrastructure - sync_data = { - 'playlist_id': sync_playlist_id, - 'playlist_name': playlist_name, - 'tracks': spotify_tracks - } - - with sync_lock: - sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} - - # Submit sync task - playlist_image_url = state['playlist'].get('image_url', '') - future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks, None, get_current_profile_id(), playlist_image_url) - active_sync_workers[sync_playlist_id] = future - - logger.info(f"Started Deezer sync for: {playlist_name} ({len(spotify_tracks)} tracks)") - return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) - - except Exception as e: - logger.error(f"Error starting Deezer sync: {e}") - return jsonify({"error": str(e)}), 500 + return _start_source_sync( + deezer_discovery_states, playlist_id, sync_id_prefix="deezer", + not_found_message="Deezer playlist not found", + not_ready_message="Deezer playlist not ready for sync", + convert_fn=convert_deezer_results_to_spotify_tracks, + name_getter=_pl_name_strict, image_getter=_pl_image_dict, + activity_label="Deezer", error_label="Deezer") @app.route('/api/deezer/sync/status/', methods=['GET']) def get_deezer_sync_status(playlist_id): @@ -22089,48 +22036,13 @@ def convert_qobuz_results_to_spotify_tracks(discovery_results): @app.route('/api/qobuz/sync/start/', methods=['POST']) def start_qobuz_sync(playlist_id): """Start sync process for a Qobuz playlist using discovered Spotify tracks.""" - try: - if playlist_id not in qobuz_discovery_states: - return jsonify({"error": "Qobuz playlist not found"}), 404 - - state = qobuz_discovery_states[playlist_id] - state['last_accessed'] = time.time() - - if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: - return jsonify({"error": "Qobuz playlist not ready for sync"}), 400 - - spotify_tracks = convert_qobuz_results_to_spotify_tracks(state['discovery_results']) - if not spotify_tracks: - return jsonify({"error": "No Spotify matches found for sync"}), 400 - - sync_playlist_id = f"qobuz_{playlist_id}" - playlist_name = state['playlist']['name'] - - add_activity_item("", "Qobuz Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") - - state['phase'] = 'syncing' - state['sync_playlist_id'] = sync_playlist_id - state['sync_progress'] = {} - - sync_data = { - 'playlist_id': sync_playlist_id, - 'playlist_name': playlist_name, - 'tracks': spotify_tracks - } - - with sync_lock: - sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} - - playlist_image_url = state['playlist'].get('image_url', '') - future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks, None, get_current_profile_id(), playlist_image_url) - active_sync_workers[sync_playlist_id] = future - - logger.info(f"Started Qobuz sync for: {playlist_name} ({len(spotify_tracks)} tracks)") - return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) - - except Exception as e: - logger.error(f"Error starting Qobuz sync: {e}") - return jsonify({"error": str(e)}), 500 + return _start_source_sync( + qobuz_discovery_states, playlist_id, sync_id_prefix="qobuz", + not_found_message="Qobuz playlist not found", + not_ready_message="Qobuz playlist not ready for sync", + convert_fn=convert_qobuz_results_to_spotify_tracks, + name_getter=_pl_name_strict, image_getter=_pl_image_dict, + activity_label="Qobuz", error_label="Qobuz") @app.route('/api/qobuz/sync/status/', methods=['GET']) @@ -22848,55 +22760,13 @@ def convert_spotify_public_results_to_spotify_tracks(discovery_results): @app.route('/api/spotify-public/sync/start/', methods=['POST']) def start_spotify_public_sync(url_hash): """Start sync process for a Spotify Public playlist using discovered Spotify tracks""" - try: - if url_hash not in spotify_public_discovery_states: - return jsonify({"error": "Spotify Public playlist not found"}), 404 - - state = spotify_public_discovery_states[url_hash] - state['last_accessed'] = time.time() - - if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: - return jsonify({"error": "Spotify Public playlist not ready for sync"}), 400 - - # Convert discovery results to Spotify tracks format - spotify_tracks = convert_spotify_public_results_to_spotify_tracks(state['discovery_results']) - - if not spotify_tracks: - return jsonify({"error": "No Spotify matches found for sync"}), 400 - - # Create a temporary playlist ID for sync tracking - sync_playlist_id = f"spotify_public_{url_hash}" - playlist_name = state['playlist']['name'] - - # Add activity for sync start - add_activity_item("", "Spotify Link Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") - - # Update Spotify Public state - state['phase'] = 'syncing' - state['sync_playlist_id'] = sync_playlist_id - state['sync_progress'] = {} - - # Start the sync using existing sync infrastructure - sync_data = { - 'playlist_id': sync_playlist_id, - 'playlist_name': playlist_name, - 'tracks': spotify_tracks - } - - with sync_lock: - sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} - - # Submit sync task - playlist_image_url = state['playlist'].get('image_url', '') - future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks, None, get_current_profile_id(), playlist_image_url) - active_sync_workers[sync_playlist_id] = future - - logger.info(f"Started Spotify Public sync for: {playlist_name} ({len(spotify_tracks)} tracks)") - return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) - - except Exception as e: - logger.error(f"Error starting Spotify Public sync: {e}") - return jsonify({"error": str(e)}), 500 + return _start_source_sync( + spotify_public_discovery_states, url_hash, sync_id_prefix="spotify_public", + not_found_message="Spotify Public playlist not found", + not_ready_message="Spotify Public playlist not ready for sync", + convert_fn=convert_spotify_public_results_to_spotify_tracks, + name_getter=_pl_name_strict, image_getter=_pl_image_dict, + activity_label="Spotify Link", error_label="Spotify Public") @app.route('/api/spotify-public/sync/status/', methods=['GET']) def get_spotify_public_sync_status(url_hash): @@ -23802,55 +23672,13 @@ def _calculate_similarity(str1, str2): @app.route('/api/youtube/sync/start/', methods=['POST']) def start_youtube_sync(url_hash): """Start sync process for a YouTube playlist using discovered Spotify tracks""" - try: - if url_hash not in youtube_playlist_states: - return jsonify({"error": "YouTube playlist not found"}), 404 - - state = youtube_playlist_states[url_hash] - state['last_accessed'] = time.time() # Update access time - - if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: - return jsonify({"error": "YouTube playlist not ready for sync"}), 400 - - # Convert discovery results to Spotify tracks format - spotify_tracks = convert_youtube_results_to_spotify_tracks(state['discovery_results']) - - if not spotify_tracks: - return jsonify({"error": "No Spotify matches found for sync"}), 400 - - # Create a temporary playlist ID for sync tracking - sync_playlist_id = f"youtube_{url_hash}" - playlist_name = state['playlist']['name'] - - # Add activity for sync start - add_activity_item("", "YouTube Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") - - # Update YouTube state - state['phase'] = 'syncing' - state['sync_playlist_id'] = sync_playlist_id - state['sync_progress'] = {} - - # Start the sync using existing sync infrastructure - sync_data = { - 'playlist_id': sync_playlist_id, - 'playlist_name': playlist_name, - 'tracks': spotify_tracks - } - - with sync_lock: - sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} - - # Submit sync task - playlist_image_url = state['playlist'].get('image_url', '') - future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks, None, get_current_profile_id(), playlist_image_url) - active_sync_workers[sync_playlist_id] = future - - logger.info(f"Started YouTube sync for: {playlist_name} ({len(spotify_tracks)} tracks)") - return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) - - except Exception as e: - logger.error(f"Error starting YouTube sync: {e}") - return jsonify({"error": str(e)}), 500 + return _start_source_sync( + youtube_playlist_states, url_hash, sync_id_prefix="youtube", + not_found_message="YouTube playlist not found", + not_ready_message="YouTube playlist not ready for sync", + convert_fn=convert_youtube_results_to_spotify_tracks, + name_getter=_pl_name_strict, image_getter=_pl_image_dict, + activity_label="YouTube", error_label="YouTube") @app.route('/api/youtube/sync/status/', methods=['GET']) def get_youtube_sync_status(url_hash): From 50ebfbd82fba5ad661a0fc6e8b565636ce31a6de Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 17:58:02 -0700 Subject: [PATCH 17/52] Discovery lift (8/N): update_*_discovery_match -> shared helper Eighth cluster, the heavyweights (~110 lines each). The fix-modal update__discovery_match for the four sources with the identical structure (Tidal, Deezer, Qobuz, Spotify-Public) -> core.discovery.endpoints.update_discovery_match(...), wired via _update_source_discovery_match. Applies the user-selected Spotify track to the discovery result (status/artist/album/duration/spotify_data/match-count) and writes the manual fix to the discovery cache. Per-source pieces are params: - source_log_label / error_label. - original_track_key ('tidal_track' / 'deezer_track' / ...). - original_artist_getter: Tidal handles string-or-object artists (first_artist_str_or_obj); the rest assume strings (first_artist_plain). - web_server helpers (join/extract artist, build_fix_modal_spotify_data, cache-key, get_database, active-discovery-source) injected. - get_json passed as a callable and invoked INSIDE the try, preserving the original's "request.get_json() inside try" behavior (malformed body -> 500). NOT folded in (genuinely divergent): iTunes-Link (saves spotify_data directly via a different cache signature), YouTube (multi-key original_track fallback), ListenBrainz (entirely different unmatch-capable structure, no cache write), Beatport. Tests: +9 (extractors; 400/404/400 guards; full happy path with result mutation + duration formatting + match-count + cache-save args; no-increment when already found; cache error swallowed; get_json raise -> 500). Full discovery suite: 198 passed. web_server.py: -400 lines. --- core/discovery/endpoints.py | 148 +++++++ tests/discovery/test_discovery_endpoints.py | 138 ++++++ web_server.py | 448 ++------------------ 3 files changed, 310 insertions(+), 424 deletions(-) diff --git a/core/discovery/endpoints.py b/core/discovery/endpoints.py index 32fe809b..db6e53aa 100644 --- a/core/discovery/endpoints.py +++ b/core/discovery/endpoints.py @@ -398,6 +398,154 @@ def get_playlist_states( return {"error": str(e)}, 500 +def first_artist_str_or_obj(original_track: Dict[str, Any]) -> str: + """Tidal: first artist from an artists list that may hold strings OR + objects ({'name': ...}); '' when empty.""" + artists = original_track.get('artists', []) + if artists: + return artists[0] if isinstance(artists[0], str) else artists[0].get('name', '') + return '' + + +def first_artist_plain(original_track: Dict[str, Any]) -> str: + """Deezer/Qobuz/Spotify-Public: first artist assuming a list of strings; + '' when empty.""" + artists = original_track.get('artists', []) + return artists[0] if artists else '' + + +def update_discovery_match( + states: Dict[str, Any], + get_json, + *, + source_log_label: str, + error_label: str, + original_track_key: str, + original_artist_getter, + join_artist_names, + extract_artist_name, + build_fix_modal_spotify_data, + get_discovery_cache_key, + get_database, + get_active_discovery_source, +) -> Tuple[Dict[str, Any], int]: + """Apply a manually-selected Spotify track to a discovery result (the + fix-modal flow) and persist it to the discovery cache. + + 1:1 lift of the ``update__discovery_match`` bodies for the four + sources with the identical structure (Tidal, Deezer, Qobuz, Spotify-Public). + Per-source pieces are params: + + - ``source_log_label`` (lowercase, e.g. "tidal") for the "Manual match + updated: ..." line; ``error_label`` for the except log. + - ``original_track_key`` — the raw-source track key on the result + ('tidal_track', 'deezer_track', ...). + - ``original_artist_getter`` — Tidal handles string-or-object artists + (``first_artist_str_or_obj``); the rest assume strings + (``first_artist_plain``). + - the web_server helpers (join/extract artist, build_fix_modal_spotify_data, + cache-key, get_database, active-discovery-source) are injected so this + stays free of those globals. + - ``get_json`` is called INSIDE the try (like the original's + ``request.get_json()``) so a malformed body yields the same 500. + + Returns ``(payload, status_code)``. + + NOT folded in: iTunes-Link (saves spotify_data directly via a different + cache signature), YouTube (multi-key original_track fallback), ListenBrainz + (entirely different unmatch-capable structure, no cache write), Beatport. + """ + try: + data = get_json() + identifier = data.get('identifier') + track_index = data.get('track_index') + spotify_track = data.get('spotify_track') + + if not identifier or track_index is None or not spotify_track: + return {'error': 'Missing required fields'}, 400 + + state = states.get(identifier) + if not state: + return {'error': 'Discovery state not found'}, 404 + + if track_index >= len(state['discovery_results']): + return {'error': 'Invalid track index'}, 400 + + result = state['discovery_results'][track_index] + old_status = result.get('status') + + result['status'] = 'Found' + result['status_class'] = 'found' + result['spotify_track'] = spotify_track['name'] + result['spotify_artist'] = join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else extract_artist_name(spotify_track['artists']) + result['spotify_album'] = spotify_track['album'] + result['spotify_id'] = spotify_track['id'] + + duration_ms = spotify_track.get('duration_ms', 0) + if duration_ms: + minutes = duration_ms // 60000 + seconds = (duration_ms % 60000) // 1000 + result['duration'] = f"{minutes}:{seconds:02d}" + else: + result['duration'] = '0:00' + + result['spotify_data'] = build_fix_modal_spotify_data(spotify_track) + result['wing_it_fallback'] = False + result['manual_match'] = True + + if old_status != 'found' and old_status != 'Found': + state['spotify_matches'] = state.get('spotify_matches', 0) + 1 + + logger.info(f"Manual match updated: {source_log_label} - {identifier} - track {track_index}") + logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") + + try: + original_track = result.get(original_track_key, {}) + original_name = original_track.get('name', spotify_track['name']) + original_artist = original_artist_getter(original_track) + + cache_key = get_discovery_cache_key(original_name, original_artist) + artists_list = spotify_track['artists'] + if isinstance(artists_list, list): + artists_list = [a if isinstance(a, str) else a.get('name', '') for a in artists_list] + image_url = spotify_track.get('image_url') or '' + album_raw = spotify_track.get('album', '') + if isinstance(album_raw, dict): + album_obj = dict(album_raw) + if image_url and not album_obj.get('image_url'): + album_obj['image_url'] = image_url + if image_url and not album_obj.get('images'): + album_obj['images'] = [{'url': image_url}] + else: + album_obj = {'name': album_raw or ''} + if image_url: + album_obj['image_url'] = image_url + album_obj['images'] = [{'url': image_url}] + + matched_data = { + 'id': spotify_track['id'], + 'name': spotify_track['name'], + 'artists': artists_list, + 'album': album_obj, + 'duration_ms': spotify_track.get('duration_ms', 0), + 'image_url': image_url, + 'source': 'spotify', + } + cache_db = get_database() + cache_db.save_discovery_cache_match( + cache_key[0], cache_key[1], get_active_discovery_source(), 1.0, matched_data, + original_name, original_artist + ) + logger.info(f"Manual fix saved to discovery cache: {original_name} by {original_artist}") + except Exception as cache_err: + logger.error(f"Error saving manual fix to discovery cache: {cache_err}") + + return {'success': True, 'result': result}, 200 + except Exception as e: + logger.error(f"Error updating {error_label} discovery match: {e}") + return {'error': str(e)}, 500 + + def start_sync( states: Dict[str, Any], key: str, diff --git a/tests/discovery/test_discovery_endpoints.py b/tests/discovery/test_discovery_endpoints.py index 219dfd77..74932087 100644 --- a/tests/discovery/test_discovery_endpoints.py +++ b/tests/discovery/test_discovery_endpoints.py @@ -652,3 +652,141 @@ def test_start_sync_exception_returns_500(): states = {'pl': {'phase': 'discovered', 'discovery_results': [1]}} body, code = start_sync(states, 'pl', **kw) assert code == 500 and "error" in body + + +# --------------------------------------------------------------------------- +# first_artist extractors +# --------------------------------------------------------------------------- + +def test_first_artist_str_or_obj(): + from core.discovery.endpoints import first_artist_str_or_obj as g + assert g({'artists': ['A', 'B']}) == 'A' + assert g({'artists': [{'name': 'Obj'}]}) == 'Obj' + assert g({'artists': []}) == '' + assert g({}) == '' + + +def test_first_artist_plain(): + from core.discovery.endpoints import first_artist_plain as g + assert g({'artists': ['A', 'B']}) == 'A' + assert g({'artists': []}) == '' + assert g({}) == '' + + +# --------------------------------------------------------------------------- +# update_discovery_match +# --------------------------------------------------------------------------- + +class _FakeCacheDB: + def __init__(self): + self.saved = [] + + def save_discovery_cache_match(self, *args): + self.saved.append(args) + + +def _update_kwargs(*, json_data, cache_db=None, getter=None): + from core.discovery.endpoints import first_artist_plain + db = cache_db or _FakeCacheDB() + kw = dict( + source_log_label='tidal', error_label='Tidal', + original_track_key='tidal_track', + original_artist_getter=getter or first_artist_plain, + join_artist_names=lambda arts: ", ".join(arts), + extract_artist_name=lambda a: str(a), + build_fix_modal_spotify_data=lambda st: {'built': st['id']}, + get_discovery_cache_key=lambda name, artist: (name.lower(), artist.lower()), + get_database=lambda: db, + get_active_discovery_source=lambda: 'spotify', + ) + return (lambda: json_data), kw, db + + +def test_update_match_missing_fields(): + from core.discovery.endpoints import update_discovery_match + gj, kw, _ = _update_kwargs(json_data={'identifier': 'p'}) # missing track_index/spotify_track + body, code = update_discovery_match({}, gj, **kw) + assert code == 400 and body == {'error': 'Missing required fields'} + + +def test_update_match_state_not_found(): + from core.discovery.endpoints import update_discovery_match + gj, kw, _ = _update_kwargs(json_data={ + 'identifier': 'p', 'track_index': 0, 'spotify_track': {'id': 'x'}}) + body, code = update_discovery_match({}, gj, **kw) + assert code == 404 and body == {'error': 'Discovery state not found'} + + +def test_update_match_invalid_index(): + from core.discovery.endpoints import update_discovery_match + gj, kw, _ = _update_kwargs(json_data={ + 'identifier': 'p', 'track_index': 5, + 'spotify_track': {'id': 'x', 'name': 'n', 'artists': [], 'album': 'a'}}) + states = {'p': {'discovery_results': []}} + body, code = update_discovery_match(states, gj, **kw) + assert code == 400 and body == {'error': 'Invalid track index'} + + +def test_update_match_happy_path_full(): + from core.discovery.endpoints import update_discovery_match + sp = {'id': 'sp9', 'name': 'New Song', 'artists': ['Art1', 'Art2'], + 'album': 'Alb', 'duration_ms': 185000, 'image_url': 'cov.jpg'} + gj, kw, db = _update_kwargs(json_data={ + 'identifier': 'p', 'track_index': 0, 'spotify_track': sp}) + result = {'status': 'not_found', 'tidal_track': {'name': 'Orig', 'artists': ['OrigArt']}} + states = {'p': {'discovery_results': [result], 'spotify_matches': 2}} + + body, code = update_discovery_match(states, gj, **kw) + + assert code == 200 and body['success'] is True + assert result['status'] == 'Found' + assert result['status_class'] == 'found' + assert result['spotify_track'] == 'New Song' + assert result['spotify_artist'] == 'Art1, Art2' + assert result['spotify_id'] == 'sp9' + assert result['duration'] == '3:05' + assert result['spotify_data'] == {'built': 'sp9'} + assert result['wing_it_fallback'] is False + assert result['manual_match'] is True + assert states['p']['spotify_matches'] == 3 # incremented (was not found) + # cache saved with normalized key + matched_data carrying image + assert len(db.saved) == 1 + key0, key1, source, score, matched, oname, oartist = db.saved[0] + assert (key0, key1) == ('orig', 'origart') + assert source == 'spotify' and score == 1.0 + assert matched['album'] == {'name': 'Alb', 'image_url': 'cov.jpg', 'images': [{'url': 'cov.jpg'}]} + assert oname == 'Orig' and oartist == 'OrigArt' + + +def test_update_match_no_increment_when_already_found(): + from core.discovery.endpoints import update_discovery_match + sp = {'id': 'x', 'name': 'n', 'artists': ['A'], 'album': 'a', 'duration_ms': 0} + gj, kw, _ = _update_kwargs(json_data={'identifier': 'p', 'track_index': 0, 'spotify_track': sp}) + result = {'status': 'Found', 'tidal_track': {}} + states = {'p': {'discovery_results': [result], 'spotify_matches': 5}} + body, code = update_discovery_match(states, gj, **kw) + assert code == 200 + assert states['p']['spotify_matches'] == 5 # unchanged + assert result['duration'] == '0:00' + + +def test_update_match_cache_error_is_swallowed(): + from core.discovery.endpoints import update_discovery_match + class _BoomDB: + def save_discovery_cache_match(self, *a): + raise RuntimeError("db down") + sp = {'id': 'x', 'name': 'n', 'artists': ['A'], 'album': 'a', 'duration_ms': 0} + gj, kw, _ = _update_kwargs(json_data={'identifier': 'p', 'track_index': 0, 'spotify_track': sp}, + cache_db=_BoomDB()) + states = {'p': {'discovery_results': [{'status': 'x', 'tidal_track': {}}], 'spotify_matches': 0}} + body, code = update_discovery_match(states, gj, **kw) + assert code == 200 and body['success'] is True # cache failure doesn't fail the request + + +def test_update_match_get_json_raises_returns_500(): + from core.discovery.endpoints import update_discovery_match + def boom(): + raise ValueError("bad json") + _, kw, _ = _update_kwargs(json_data={}) + body, code = update_discovery_match({}, boom, **kw) + assert code == 500 and 'error' in body diff --git a/web_server.py b/web_server.py index 7540989a..de9809dd 100644 --- a/web_server.py +++ b/web_server.py @@ -20566,117 +20566,7 @@ def get_tidal_discovery_status(playlist_id): @app.route('/api/tidal/discovery/update_match', methods=['POST']) def update_tidal_discovery_match(): """Update a Tidal discovery result with manually selected Spotify track""" - try: - data = request.get_json() - identifier = data.get('identifier') # playlist_id - track_index = data.get('track_index') - spotify_track = data.get('spotify_track') - - if not identifier or track_index is None or not spotify_track: - return jsonify({'error': 'Missing required fields'}), 400 - - # Get the state - state = tidal_discovery_states.get(identifier) - - if not state: - return jsonify({'error': 'Discovery state not found'}), 404 - - if track_index >= len(state['discovery_results']): - return jsonify({'error': 'Invalid track index'}), 400 - - # Update the result - result = state['discovery_results'][track_index] - old_status = result.get('status') - - # Update with user-selected track - result['status'] = 'Found' - result['status_class'] = 'found' - result['spotify_track'] = spotify_track['name'] - result['spotify_artist'] = _join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else _extract_artist_name(spotify_track['artists']) - result['spotify_album'] = spotify_track['album'] - result['spotify_id'] = spotify_track['id'] - - # Format duration (Tidal doesn't show duration in table, but store it anyway) - duration_ms = spotify_track.get('duration_ms', 0) - if duration_ms: - minutes = duration_ms // 60000 - seconds = (duration_ms % 60000) // 1000 - result['duration'] = f"{minutes}:{seconds:02d}" - else: - result['duration'] = '0:00' - - # IMPORTANT: Also set spotify_data for sync/download compatibility. - # Manual match from the fix modal — build a rich spotify_data (album - # as dict with image info) matching the normal discovery shape, and - # explicitly clear any prior wing-it flag since the user picked a - # real metadata match. - result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) - result['wing_it_fallback'] = False - - result['manual_match'] = True # Flag for tracking - - # Update match count if status changed from not found/error - if old_status != 'found' and old_status != 'Found': - state['spotify_matches'] = state.get('spotify_matches', 0) + 1 - - logger.info(f"Manual match updated: tidal - {identifier} - track {track_index}") - logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") - - # Save manual fix to discovery cache so it appears in discovery pool - try: - original_track = result.get('tidal_track', {}) - original_name = original_track.get('name', spotify_track['name']) - original_artist = '' - original_artists = original_track.get('artists', []) - if original_artists: - original_artist = original_artists[0] if isinstance(original_artists[0], str) else original_artists[0].get('name', '') - - cache_key = _get_discovery_cache_key(original_name, original_artist) - # Normalize artists to plain strings for cache consistency - artists_list = spotify_track['artists'] - if isinstance(artists_list, list): - artists_list = [a if isinstance(a, str) else a.get('name', '') for a in artists_list] - # Preserve cover image info so the download pipeline can find - # artwork when this cached match is used later. The fix modal - # sends image_url at the top level; search results often return - # album as a bare string, which previously dropped the artwork. - image_url = spotify_track.get('image_url') or '' - album_raw = spotify_track.get('album', '') - if isinstance(album_raw, dict): - album_obj = dict(album_raw) - if image_url and not album_obj.get('image_url'): - album_obj['image_url'] = image_url - if image_url and not album_obj.get('images'): - album_obj['images'] = [{'url': image_url}] - else: - album_obj = {'name': album_raw or ''} - if image_url: - album_obj['image_url'] = image_url - album_obj['images'] = [{'url': image_url}] - - matched_data = { - 'id': spotify_track['id'], - 'name': spotify_track['name'], - 'artists': artists_list, - 'album': album_obj, - 'duration_ms': spotify_track.get('duration_ms', 0), - 'image_url': image_url, - 'source': 'spotify', - } - cache_db = get_database() - cache_db.save_discovery_cache_match( - cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, - original_name, original_artist - ) - logger.info(f"Manual fix saved to discovery cache: {original_name} by {original_artist}") - except Exception as cache_err: - logger.error(f"Error saving manual fix to discovery cache: {cache_err}") - - return jsonify({'success': True, 'result': result}) - - except Exception as e: - logger.error(f"Error updating Tidal discovery match: {e}") - return jsonify({'error': str(e)}), 500 + return _update_source_discovery_match(tidal_discovery_states, "tidal", "Tidal", "tidal_track", _first_artist_str_or_obj) @app.route('/api/tidal/playlists/states', methods=['GET']) @@ -20988,12 +20878,15 @@ from core.discovery.endpoints import ( reset_playlist as _reset_playlist_core, get_playlist_states as _get_playlist_states_core, start_sync as _start_sync_core, + update_discovery_match as _update_discovery_match_core, playlist_name_attr_or_unknown as _pl_name_attr_or_unknown, playlist_name_strict as _pl_name_strict, playlist_name_safe as _pl_name_safe, playlist_name_obj as _pl_name_obj, playlist_image_obj as _pl_image_obj, playlist_image_dict as _pl_image_dict, + first_artist_str_or_obj as _first_artist_str_or_obj, + first_artist_plain as _first_artist_plain, ) @@ -21082,6 +20975,22 @@ def _start_source_sync(states, key, *, sync_id_prefix, not_found_message, return jsonify(body), code +def _update_source_discovery_match(states, source_log_label, error_label, + original_track_key, artist_getter): + """Thin glue for the per-source update_*_discovery_match (fix-modal) routes + (Tidal/Deezer/Qobuz/Spotify-Public) — injects the web_server helpers.""" + body, code = _update_discovery_match_core( + states, lambda: request.get_json(), + source_log_label=source_log_label, error_label=error_label, + original_track_key=original_track_key, original_artist_getter=artist_getter, + join_artist_names=_join_artist_names, extract_artist_name=_extract_artist_name, + build_fix_modal_spotify_data=_build_fix_modal_spotify_data, + get_discovery_cache_key=_get_discovery_cache_key, get_database=get_database, + get_active_discovery_source=_get_active_discovery_source, + ) + return jsonify(body), code + + def _build_tidal_discovery_deps(): """Build the TidalDiscoveryDeps bundle from web_server.py globals on each call.""" return _discovery_tidal.TidalDiscoveryDeps( @@ -21364,115 +21273,7 @@ def get_deezer_discovery_status(playlist_id): @app.route('/api/deezer/discovery/update_match', methods=['POST']) def update_deezer_discovery_match(): """Update a Deezer discovery result with manually selected Spotify track""" - try: - data = request.get_json() - identifier = data.get('identifier') # playlist_id - track_index = data.get('track_index') - spotify_track = data.get('spotify_track') - - if not identifier or track_index is None or not spotify_track: - return jsonify({'error': 'Missing required fields'}), 400 - - # Get the state - state = deezer_discovery_states.get(identifier) - - if not state: - return jsonify({'error': 'Discovery state not found'}), 404 - - if track_index >= len(state['discovery_results']): - return jsonify({'error': 'Invalid track index'}), 400 - - # Update the result - result = state['discovery_results'][track_index] - old_status = result.get('status') - - # Update with user-selected track - result['status'] = 'Found' - result['status_class'] = 'found' - result['spotify_track'] = spotify_track['name'] - result['spotify_artist'] = _join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else _extract_artist_name(spotify_track['artists']) - result['spotify_album'] = spotify_track['album'] - result['spotify_id'] = spotify_track['id'] - - # Format duration - duration_ms = spotify_track.get('duration_ms', 0) - if duration_ms: - minutes = duration_ms // 60000 - seconds = (duration_ms % 60000) // 1000 - result['duration'] = f"{minutes}:{seconds:02d}" - else: - result['duration'] = '0:00' - - # IMPORTANT: Also set spotify_data for sync/download compatibility. - # Manual match from the fix modal — build a rich spotify_data (album - # as dict with image info) matching the normal discovery shape, and - # explicitly clear any prior wing-it flag since the user picked a - # real metadata match. - result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) - result['wing_it_fallback'] = False - - result['manual_match'] = True - - # Update match count if status changed from not found/error - if old_status != 'found' and old_status != 'Found': - state['spotify_matches'] = state.get('spotify_matches', 0) + 1 - - logger.info(f"Manual match updated: deezer - {identifier} - track {track_index}") - logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") - - # Save manual fix to discovery cache so it appears in discovery pool - try: - original_track = result.get('deezer_track', {}) - original_name = original_track.get('name', spotify_track['name']) - original_artists = original_track.get('artists', []) - original_artist = original_artists[0] if original_artists else '' - - cache_key = _get_discovery_cache_key(original_name, original_artist) - # Normalize artists to plain strings for cache consistency - artists_list = spotify_track['artists'] - if isinstance(artists_list, list): - artists_list = [a if isinstance(a, str) else a.get('name', '') for a in artists_list] - # Preserve cover image info so the download pipeline can find - # artwork when this cached match is used later. The fix modal - # sends image_url at the top level; search results often return - # album as a bare string, which previously dropped the artwork. - image_url = spotify_track.get('image_url') or '' - album_raw = spotify_track.get('album', '') - if isinstance(album_raw, dict): - album_obj = dict(album_raw) - if image_url and not album_obj.get('image_url'): - album_obj['image_url'] = image_url - if image_url and not album_obj.get('images'): - album_obj['images'] = [{'url': image_url}] - else: - album_obj = {'name': album_raw or ''} - if image_url: - album_obj['image_url'] = image_url - album_obj['images'] = [{'url': image_url}] - - matched_data = { - 'id': spotify_track['id'], - 'name': spotify_track['name'], - 'artists': artists_list, - 'album': album_obj, - 'duration_ms': spotify_track.get('duration_ms', 0), - 'image_url': image_url, - 'source': 'spotify', - } - cache_db = get_database() - cache_db.save_discovery_cache_match( - cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, - original_name, original_artist - ) - logger.info(f"Manual fix saved to discovery cache: {original_name} by {original_artist}") - except Exception as cache_err: - logger.error(f"Error saving manual fix to discovery cache: {cache_err}") - - return jsonify({'success': True, 'result': result}) - - except Exception as e: - logger.error(f"Error updating Deezer discovery match: {e}") - return jsonify({'error': str(e)}), 500 + return _update_source_discovery_match(deezer_discovery_states, "deezer", "Deezer", "deezer_track", _first_artist_plain) @app.route('/api/deezer/playlists/states', methods=['GET']) def get_deezer_playlist_states(): @@ -21814,101 +21615,8 @@ def get_qobuz_discovery_status(playlist_id): @app.route('/api/qobuz/discovery/update_match', methods=['POST']) def update_qobuz_discovery_match(): - """Update a Qobuz discovery result with a manually selected Spotify track.""" - try: - data = request.get_json() - identifier = data.get('identifier') # playlist_id - track_index = data.get('track_index') - spotify_track = data.get('spotify_track') - - if not identifier or track_index is None or not spotify_track: - return jsonify({'error': 'Missing required fields'}), 400 - - state = qobuz_discovery_states.get(identifier) - if not state: - return jsonify({'error': 'Discovery state not found'}), 404 - - if track_index >= len(state['discovery_results']): - return jsonify({'error': 'Invalid track index'}), 400 - - result = state['discovery_results'][track_index] - old_status = result.get('status') - - result['status'] = 'Found' - result['status_class'] = 'found' - result['spotify_track'] = spotify_track['name'] - result['spotify_artist'] = _join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else _extract_artist_name(spotify_track['artists']) - result['spotify_album'] = spotify_track['album'] - result['spotify_id'] = spotify_track['id'] - - duration_ms = spotify_track.get('duration_ms', 0) - if duration_ms: - minutes = duration_ms // 60000 - seconds = (duration_ms % 60000) // 1000 - result['duration'] = f"{minutes}:{seconds:02d}" - else: - result['duration'] = '0:00' - - # Manual match from the fix modal — build rich spotify_data matching - # the normal discovery shape, clear wing-it flag since the user - # picked a real metadata match. - result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) - result['wing_it_fallback'] = False - result['manual_match'] = True - - if old_status != 'found' and old_status != 'Found': - state['spotify_matches'] = state.get('spotify_matches', 0) + 1 - - logger.info(f"Manual match updated: qobuz - {identifier} - track {track_index}") - logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") - - try: - original_track = result.get('qobuz_track', {}) - original_name = original_track.get('name', spotify_track['name']) - original_artists = original_track.get('artists', []) - original_artist = original_artists[0] if original_artists else '' - - cache_key = _get_discovery_cache_key(original_name, original_artist) - artists_list = spotify_track['artists'] - if isinstance(artists_list, list): - artists_list = [a if isinstance(a, str) else a.get('name', '') for a in artists_list] - image_url = spotify_track.get('image_url') or '' - album_raw = spotify_track.get('album', '') - if isinstance(album_raw, dict): - album_obj = dict(album_raw) - if image_url and not album_obj.get('image_url'): - album_obj['image_url'] = image_url - if image_url and not album_obj.get('images'): - album_obj['images'] = [{'url': image_url}] - else: - album_obj = {'name': album_raw or ''} - if image_url: - album_obj['image_url'] = image_url - album_obj['images'] = [{'url': image_url}] - - matched_data = { - 'id': spotify_track['id'], - 'name': spotify_track['name'], - 'artists': artists_list, - 'album': album_obj, - 'duration_ms': spotify_track.get('duration_ms', 0), - 'image_url': image_url, - 'source': 'spotify', - } - cache_db = get_database() - cache_db.save_discovery_cache_match( - cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, - original_name, original_artist - ) - logger.info(f"Manual fix saved to discovery cache: {original_name} by {original_artist}") - except Exception as cache_err: - logger.error(f"Error saving manual fix to discovery cache: {cache_err}") - - return jsonify({'success': True, 'result': result}) - - except Exception as e: - logger.error(f"Error updating Qobuz discovery match: {e}") - return jsonify({'error': str(e)}), 500 + """Update a Qobuz discovery result with manually selected Spotify track""" + return _update_source_discovery_match(qobuz_discovery_states, "qobuz", "Qobuz", "qobuz_track", _first_artist_plain) @app.route('/api/qobuz/playlists/states', methods=['GET']) @@ -22524,115 +22232,7 @@ def get_spotify_public_discovery_status(url_hash): @app.route('/api/spotify-public/discovery/update_match', methods=['POST']) def update_spotify_public_discovery_match(): """Update a Spotify Public discovery result with manually selected Spotify track""" - try: - data = request.get_json() - identifier = data.get('identifier') # url_hash - track_index = data.get('track_index') - spotify_track = data.get('spotify_track') - - if not identifier or track_index is None or not spotify_track: - return jsonify({'error': 'Missing required fields'}), 400 - - # Get the state - state = spotify_public_discovery_states.get(identifier) - - if not state: - return jsonify({'error': 'Discovery state not found'}), 404 - - if track_index >= len(state['discovery_results']): - return jsonify({'error': 'Invalid track index'}), 400 - - # Update the result - result = state['discovery_results'][track_index] - old_status = result.get('status') - - # Update with user-selected track - result['status'] = 'Found' - result['status_class'] = 'found' - result['spotify_track'] = spotify_track['name'] - result['spotify_artist'] = _join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else _extract_artist_name(spotify_track['artists']) - result['spotify_album'] = spotify_track['album'] - result['spotify_id'] = spotify_track['id'] - - # Format duration - duration_ms = spotify_track.get('duration_ms', 0) - if duration_ms: - minutes = duration_ms // 60000 - seconds = (duration_ms % 60000) // 1000 - result['duration'] = f"{minutes}:{seconds:02d}" - else: - result['duration'] = '0:00' - - # IMPORTANT: Also set spotify_data for sync/download compatibility. - # Manual match from the fix modal — build a rich spotify_data (album - # as dict with image info) matching the normal discovery shape, and - # explicitly clear any prior wing-it flag since the user picked a - # real metadata match. - result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) - result['wing_it_fallback'] = False - - result['manual_match'] = True - - # Update match count if status changed from not found/error - if old_status != 'found' and old_status != 'Found': - state['spotify_matches'] = state.get('spotify_matches', 0) + 1 - - logger.info(f"Manual match updated: spotify_public - {identifier} - track {track_index}") - logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") - - # Save manual fix to discovery cache so it appears in discovery pool - try: - original_track = result.get('spotify_public_track', {}) - original_name = original_track.get('name', spotify_track['name']) - original_artists = original_track.get('artists', []) - original_artist = original_artists[0] if original_artists else '' - - cache_key = _get_discovery_cache_key(original_name, original_artist) - # Normalize artists to plain strings for cache consistency - artists_list = spotify_track['artists'] - if isinstance(artists_list, list): - artists_list = [a if isinstance(a, str) else a.get('name', '') for a in artists_list] - # Preserve cover image info so the download pipeline can find - # artwork when this cached match is used later. The fix modal - # sends image_url at the top level; search results often return - # album as a bare string, which previously dropped the artwork. - image_url = spotify_track.get('image_url') or '' - album_raw = spotify_track.get('album', '') - if isinstance(album_raw, dict): - album_obj = dict(album_raw) - if image_url and not album_obj.get('image_url'): - album_obj['image_url'] = image_url - if image_url and not album_obj.get('images'): - album_obj['images'] = [{'url': image_url}] - else: - album_obj = {'name': album_raw or ''} - if image_url: - album_obj['image_url'] = image_url - album_obj['images'] = [{'url': image_url}] - - matched_data = { - 'id': spotify_track['id'], - 'name': spotify_track['name'], - 'artists': artists_list, - 'album': album_obj, - 'duration_ms': spotify_track.get('duration_ms', 0), - 'image_url': image_url, - 'source': 'spotify', - } - cache_db = get_database() - cache_db.save_discovery_cache_match( - cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, - original_name, original_artist - ) - logger.info(f"Manual fix saved to discovery cache: {original_name} by {original_artist}") - except Exception as cache_err: - logger.error(f"Error saving manual fix to discovery cache: {cache_err}") - - return jsonify({'success': True, 'result': result}) - - except Exception as e: - logger.error(f"Error updating Spotify Public discovery match: {e}") - return jsonify({'error': str(e)}), 500 + return _update_source_discovery_match(spotify_public_discovery_states, "spotify_public", "Spotify Public", "spotify_public_track", _first_artist_plain) @app.route('/api/spotify-public/playlists/states', methods=['GET']) def get_spotify_public_playlist_states(): From 4caf36deb1f0ac5ea9a4382acacd3f1a5367fbfe Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 18:07:16 -0700 Subject: [PATCH 18/52] Discovery lift (9/N): update_*_playlist_phase -> shared helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ninth cluster: update__playlist_phase for the five sources sharing the identical validation + full-message response (Tidal, Deezer, Qobuz, Spotify-Public, YouTube) -> core.discovery.endpoints.update_playlist_phase(...), wired via _update_source_playlist_phase + the _PHASE_LIST/_PHASE_LIST_YT constants. Per-source params: - valid_phases — YouTube additionally allows 'parsed'. - apply_extra_fields — Deezer/Qobuz/Spotify-Public also persist download_process_id / converted_spotify_playlist_id from the body; Tidal and YouTube do NOT, so they pass False (kept strictly 1:1 — the generic won't apply those keys for them even if a caller sent them). - not_found_message / error_label; get_json invoked inside the try. NOT folded in: iTunes-Link — uses data.get('phase') (no "Phase not provided" 400) and returns a no-message payload. Tests: +7 (404, missing-phase 400, invalid 400, happy path with extra-fields suppressed, extra-fields applied when enabled, YouTube 'parsed' allowed, exception -> 500). Full discovery suite: 205 passed. web_server.py: -123 lines. --- core/discovery/endpoints.py | 59 +++++++ tests/discovery/test_discovery_endpoints.py | 87 ++++++++++ web_server.py | 169 +++----------------- 3 files changed, 169 insertions(+), 146 deletions(-) diff --git a/core/discovery/endpoints.py b/core/discovery/endpoints.py index db6e53aa..5b2f1312 100644 --- a/core/discovery/endpoints.py +++ b/core/discovery/endpoints.py @@ -398,6 +398,65 @@ def get_playlist_states( return {"error": str(e)}, 500 +def update_playlist_phase( + states: Dict[str, Any], + key: str, + get_json, + *, + not_found_message: str, + error_label: str, + valid_phases: List[str], + apply_extra_fields: bool, +) -> Tuple[Dict[str, Any], int]: + """Update a discovery playlist's phase (used when the modal closes, e.g. to + reset download_complete -> discovered). + + 1:1 lift of the ``update__playlist_phase`` bodies for the five + sources with the identical validation + full-message response (Tidal, + Deezer, Qobuz, Spotify-Public, YouTube). Per-source params: + + - ``valid_phases`` — YouTube's list additionally includes 'parsed'. + - ``apply_extra_fields`` — Deezer/Qobuz/Spotify-Public also persist + download_process_id / converted_spotify_playlist_id from the body; + Tidal/YouTube do NOT (so pass False to keep them 1:1). + - ``not_found_message`` / ``error_label``; ``get_json`` invoked inside the + try like the original ``request.get_json()``. + + Returns ``(payload, status_code)``. + + NOT folded in: iTunes-Link — it uses ``data.get('phase')`` (no separate + "Phase not provided" 400) and returns a no-message payload. + """ + try: + if key not in states: + return {"error": not_found_message}, 404 + + data = get_json() + if not data or 'phase' not in data: + return {"error": "Phase not provided"}, 400 + + new_phase = data['phase'] + if new_phase not in valid_phases: + return {"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}, 400 + + state = states[key] + old_phase = state.get('phase', 'unknown') + state['phase'] = new_phase + state['last_accessed'] = time.time() + + if apply_extra_fields: + if 'download_process_id' in data: + state['download_process_id'] = data['download_process_id'] + if 'converted_spotify_playlist_id' in data: + state['converted_spotify_playlist_id'] = data['converted_spotify_playlist_id'] + + logger.info(f"Updated {error_label} playlist {key} phase: {old_phase} → {new_phase}") + return {"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase}, 200 + except Exception as e: + logger.error(f"Error updating {error_label} playlist phase: {e}") + return {"error": str(e)}, 500 + + def first_artist_str_or_obj(original_track: Dict[str, Any]) -> str: """Tidal: first artist from an artists list that may hold strings OR objects ({'name': ...}); '' when empty.""" diff --git a/tests/discovery/test_discovery_endpoints.py b/tests/discovery/test_discovery_endpoints.py index 74932087..618e18cc 100644 --- a/tests/discovery/test_discovery_endpoints.py +++ b/tests/discovery/test_discovery_endpoints.py @@ -790,3 +790,90 @@ def test_update_match_get_json_raises_returns_500(): _, kw, _ = _update_kwargs(json_data={}) body, code = update_discovery_match({}, boom, **kw) assert code == 500 and 'error' in body + + +# --------------------------------------------------------------------------- +# update_playlist_phase +# --------------------------------------------------------------------------- + +_PHASES = ['fresh', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] + + +def test_update_phase_not_found(): + from core.discovery.endpoints import update_playlist_phase + body, code = update_playlist_phase({}, 'k', lambda: {'phase': 'fresh'}, + not_found_message='Tidal playlist not found', + error_label='Tidal', valid_phases=_PHASES, + apply_extra_fields=False) + assert code == 404 and body == {"error": "Tidal playlist not found"} + + +def test_update_phase_missing_phase(): + from core.discovery.endpoints import update_playlist_phase + states = {'k': {'phase': 'discovered'}} + body, code = update_playlist_phase(states, 'k', lambda: {}, + not_found_message='nf', error_label='Tidal', + valid_phases=_PHASES, apply_extra_fields=False) + assert code == 400 and body == {"error": "Phase not provided"} + + +def test_update_phase_invalid(): + from core.discovery.endpoints import update_playlist_phase + states = {'k': {'phase': 'discovered'}} + body, code = update_playlist_phase(states, 'k', lambda: {'phase': 'bogus'}, + not_found_message='nf', error_label='Tidal', + valid_phases=_PHASES, apply_extra_fields=False) + assert code == 400 and 'Invalid phase' in body['error'] + + +def test_update_phase_happy_no_extra_fields(): + from core.discovery.endpoints import update_playlist_phase + state = {'phase': 'download_complete', 'last_accessed': 0} + body, code = update_playlist_phase({'k': state}, 'k', + lambda: {'phase': 'discovered', + 'download_process_id': 'dp'}, + not_found_message='nf', error_label='Tidal', + valid_phases=_PHASES, apply_extra_fields=False) + assert code == 200 + assert body == {"success": True, "message": "Phase updated to discovered", + "old_phase": "download_complete", "new_phase": "discovered"} + assert state['phase'] == 'discovered' + assert state['last_accessed'] != 0 + # apply_extra_fields=False -> download_process_id NOT applied (1:1 for Tidal/YouTube) + assert 'download_process_id' not in state + + +def test_update_phase_applies_extra_fields_when_enabled(): + from core.discovery.endpoints import update_playlist_phase + state = {'phase': 'discovered'} + body, code = update_playlist_phase({'k': state}, 'k', + lambda: {'phase': 'downloading', + 'download_process_id': 'dp7', + 'converted_spotify_playlist_id': 'cv7'}, + not_found_message='nf', error_label='Deezer', + valid_phases=_PHASES, apply_extra_fields=True) + assert code == 200 + assert state['download_process_id'] == 'dp7' + assert state['converted_spotify_playlist_id'] == 'cv7' + + +def test_update_phase_youtube_parsed_allowed(): + from core.discovery.endpoints import update_playlist_phase + yt_phases = ['fresh', 'parsed', 'discovering', 'discovered', 'syncing', + 'sync_complete', 'downloading', 'download_complete'] + state = {'phase': 'discovered'} + body, code = update_playlist_phase({'k': state}, 'k', lambda: {'phase': 'parsed'}, + not_found_message='nf', error_label='YouTube', + valid_phases=yt_phases, apply_extra_fields=False) + assert code == 200 and state['phase'] == 'parsed' + + +def test_update_phase_exception_500(): + from core.discovery.endpoints import update_playlist_phase + def boom(): + raise ValueError('bad') + states = {'k': {'phase': 'discovered'}} + body, code = update_playlist_phase(states, 'k', boom, not_found_message='nf', + error_label='Tidal', valid_phases=_PHASES, + apply_extra_fields=False) + assert code == 500 and 'error' in body diff --git a/web_server.py b/web_server.py index de9809dd..a4cc81e5 100644 --- a/web_server.py +++ b/web_server.py @@ -20620,31 +20620,7 @@ def delete_tidal_playlist(playlist_id): @app.route('/api/tidal/update_phase/', methods=['POST']) def update_tidal_playlist_phase(playlist_id): """Update Tidal playlist phase (used when modal closes to reset from download_complete to discovered)""" - try: - if playlist_id not in tidal_discovery_states: - return jsonify({"error": "Tidal playlist not found"}), 404 - - data = request.get_json() - if not data or 'phase' not in data: - return jsonify({"error": "Phase not provided"}), 400 - - new_phase = data['phase'] - valid_phases = ['fresh', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] - - if new_phase not in valid_phases: - return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400 - - state = tidal_discovery_states[playlist_id] - old_phase = state.get('phase', 'unknown') - state['phase'] = new_phase - state['last_accessed'] = time.time() - - logger.info(f"Updated Tidal playlist {playlist_id} phase: {old_phase} → {new_phase}") - return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase}) - - except Exception as e: - logger.error(f"Error updating Tidal playlist phase: {e}") - return jsonify({"error": str(e)}), 500 + return _update_source_playlist_phase(tidal_discovery_states, playlist_id, "Tidal playlist not found", "Tidal", _PHASE_LIST, False) _playlist_discovery_cancelled = set() # Set of automation_ids that have been cancelled @@ -20879,6 +20855,7 @@ from core.discovery.endpoints import ( get_playlist_states as _get_playlist_states_core, start_sync as _start_sync_core, update_discovery_match as _update_discovery_match_core, + update_playlist_phase as _update_playlist_phase_core, playlist_name_attr_or_unknown as _pl_name_attr_or_unknown, playlist_name_strict as _pl_name_strict, playlist_name_safe as _pl_name_safe, @@ -20991,6 +20968,23 @@ def _update_source_discovery_match(states, source_log_label, error_label, return jsonify(body), code +# Valid phase lists for update_*_playlist_phase (YouTube additionally allows 'parsed'). +_PHASE_LIST = ['fresh', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] +_PHASE_LIST_YT = ['fresh', 'parsed', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] + + +def _update_source_playlist_phase(states, key, not_found_message, error_label, + valid_phases, apply_extra_fields): + """Thin glue for the per-source update_*_playlist_phase routes + (Tidal/Deezer/Qobuz/Spotify-Public/YouTube).""" + body, code = _update_playlist_phase_core( + states, key, lambda: request.get_json(), + not_found_message=not_found_message, error_label=error_label, + valid_phases=valid_phases, apply_extra_fields=apply_extra_fields, + ) + return jsonify(body), code + + def _build_tidal_discovery_deps(): """Build the TidalDiscoveryDeps bundle from web_server.py globals on each call.""" return _discovery_tidal.TidalDiscoveryDeps( @@ -21326,39 +21320,7 @@ def delete_deezer_playlist(playlist_id): @app.route('/api/deezer/update_phase/', methods=['POST']) def update_deezer_playlist_phase(playlist_id): """Update Deezer playlist phase (used when modal closes to reset from download_complete to discovered)""" - try: - if playlist_id not in deezer_discovery_states: - return jsonify({"error": "Deezer playlist not found"}), 404 - - data = request.get_json() - if not data or 'phase' not in data: - return jsonify({"error": "Phase not provided"}), 400 - - new_phase = data['phase'] - valid_phases = ['fresh', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] - - if new_phase not in valid_phases: - return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400 - - state = deezer_discovery_states[playlist_id] - old_phase = state.get('phase', 'unknown') - state['phase'] = new_phase - state['last_accessed'] = time.time() - - # Update download process ID if provided (for download persistence) - if 'download_process_id' in data: - state['download_process_id'] = data['download_process_id'] - - # Update converted Spotify playlist ID if provided (for download persistence) - if 'converted_spotify_playlist_id' in data: - state['converted_spotify_playlist_id'] = data['converted_spotify_playlist_id'] - - logger.info(f"Updated Deezer playlist {playlist_id} phase: {old_phase} → {new_phase}") - return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase}) - - except Exception as e: - logger.error(f"Error updating Deezer playlist phase: {e}") - return jsonify({"error": str(e)}), 500 + return _update_source_playlist_phase(deezer_discovery_states, playlist_id, "Deezer playlist not found", "Deezer", _PHASE_LIST, True) # Deezer discovery worker logic lives in core/discovery/deezer.py. @@ -21673,36 +21635,7 @@ def delete_qobuz_playlist(playlist_id): @app.route('/api/qobuz/update_phase/', methods=['POST']) def update_qobuz_playlist_phase(playlist_id): """Update Qobuz playlist phase (used when modal closes to reset from download_complete to discovered).""" - try: - if playlist_id not in qobuz_discovery_states: - return jsonify({"error": "Qobuz playlist not found"}), 404 - - data = request.get_json() - if not data or 'phase' not in data: - return jsonify({"error": "Phase not provided"}), 400 - - new_phase = data['phase'] - valid_phases = ['fresh', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] - - if new_phase not in valid_phases: - return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400 - - state = qobuz_discovery_states[playlist_id] - old_phase = state.get('phase', 'unknown') - state['phase'] = new_phase - state['last_accessed'] = time.time() - - if 'download_process_id' in data: - state['download_process_id'] = data['download_process_id'] - if 'converted_spotify_playlist_id' in data: - state['converted_spotify_playlist_id'] = data['converted_spotify_playlist_id'] - - logger.info(f"Updated Qobuz playlist {playlist_id} phase: {old_phase} → {new_phase}") - return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase}) - - except Exception as e: - logger.error(f"Error updating Qobuz playlist phase: {e}") - return jsonify({"error": str(e)}), 500 + return _update_source_playlist_phase(qobuz_discovery_states, playlist_id, "Qobuz playlist not found", "Qobuz", _PHASE_LIST, True) # Qobuz discovery worker logic lives in core/discovery/qobuz.py. @@ -22284,39 +22217,7 @@ def delete_spotify_public_playlist(url_hash): @app.route('/api/spotify-public/update_phase/', methods=['POST']) def update_spotify_public_playlist_phase(url_hash): """Update Spotify Public playlist phase (used when modal closes to reset from download_complete to discovered)""" - try: - if url_hash not in spotify_public_discovery_states: - return jsonify({"error": "Spotify Public playlist not found"}), 404 - - data = request.get_json() - if not data or 'phase' not in data: - return jsonify({"error": "Phase not provided"}), 400 - - new_phase = data['phase'] - valid_phases = ['fresh', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] - - if new_phase not in valid_phases: - return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400 - - state = spotify_public_discovery_states[url_hash] - old_phase = state.get('phase', 'unknown') - state['phase'] = new_phase - state['last_accessed'] = time.time() - - # Update download process ID if provided (for download persistence) - if 'download_process_id' in data: - state['download_process_id'] = data['download_process_id'] - - # Update converted Spotify playlist ID if provided (for download persistence) - if 'converted_spotify_playlist_id' in data: - state['converted_spotify_playlist_id'] = data['converted_spotify_playlist_id'] - - logger.info(f"Updated Spotify Public playlist {url_hash} phase: {old_phase} → {new_phase}") - return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase}) - - except Exception as e: - logger.error(f"Error updating Spotify Public playlist phase: {e}") - return jsonify({"error": str(e)}), 500 + return _update_source_playlist_phase(spotify_public_discovery_states, url_hash, "Spotify Public playlist not found", "Spotify Public", _PHASE_LIST, True) # Spotify Public discovery worker logic lives in core/discovery/spotify_public.py. @@ -23423,31 +23324,7 @@ def delete_youtube_playlist(url_hash): @app.route('/api/youtube/update_phase/', methods=['POST']) def update_youtube_playlist_phase(url_hash): """Update YouTube playlist phase (used when modal closes to reset from download_complete to discovered)""" - try: - if url_hash not in youtube_playlist_states: - return jsonify({"error": "YouTube playlist not found"}), 404 - - data = request.get_json() - if not data or 'phase' not in data: - return jsonify({"error": "Phase not provided"}), 400 - - new_phase = data['phase'] - valid_phases = ['fresh', 'parsed', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] - - if new_phase not in valid_phases: - return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400 - - state = youtube_playlist_states[url_hash] - old_phase = state.get('phase', 'unknown') - state['phase'] = new_phase - state['last_accessed'] = time.time() - - logger.info(f"Updated YouTube playlist {url_hash} phase: {old_phase} → {new_phase}") - return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase}) - - except Exception as e: - logger.error(f"Error updating YouTube playlist phase: {e}") - return jsonify({"error": str(e)}), 500 + return _update_source_playlist_phase(youtube_playlist_states, url_hash, "YouTube playlist not found", "YouTube", _PHASE_LIST_YT, False) def convert_youtube_results_to_spotify_tracks(discovery_results): """Convert YouTube discovery results to Spotify tracks format for sync""" From d5f6a14ba11b2263a280937173479b28982598d9 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 18:17:36 -0700 Subject: [PATCH 19/52] Discovery lift (10/N): save_*_bubble_snapshot -> shared helper Final cluster: the four structurally-identical snapshot endpoints (discover_downloads, artist_bubbles, search_bubbles, beatport_bubbles) -> core.discovery.endpoints.save_bubble_snapshot(...), wired via _save_source_bubble_snapshot. All four validate a payload key, persist via db.save_bubble_snapshot(kind, items, profile_id=...), and return a count + timestamp; they differ only by: - payload_key ('downloads' for discover, 'bubbles' for the rest) + its no_data_error message. - snapshot_kind, success_noun, and the info/except log subject + noun ("downloads"/"artists"/"albums/tracks"/"charts"). get_database / get_current_profile_id injected; get_json (request.json) invoked inside the try, preserving the original 400/500 behavior incl. traceback dump. Tests: +5 (missing key 400, None body 400, happy path with kind/profile/count/ timestamp, discover_downloads variant, exception -> 500). Full discovery suite: 210 passed. web_server.py: -98 lines. --- core/discovery/endpoints.py | 55 ++++++++ tests/discovery/test_discovery_endpoints.py | 78 ++++++++++++ web_server.py | 134 +++----------------- 3 files changed, 151 insertions(+), 116 deletions(-) diff --git a/core/discovery/endpoints.py b/core/discovery/endpoints.py index 5b2f1312..69dbb0d8 100644 --- a/core/discovery/endpoints.py +++ b/core/discovery/endpoints.py @@ -398,6 +398,61 @@ def get_playlist_states( return {"error": str(e)}, 500 +def save_bubble_snapshot( + get_json, + *, + payload_key: str, + no_data_error: str, + snapshot_kind: str, + success_noun: str, + log_subject: str, + log_noun: str, + get_database, + get_current_profile_id, +) -> Tuple[Dict[str, Any], int]: + """Persist a bubble/download snapshot for cross-refresh hydration. + + 1:1 lift of the four structurally-identical snapshot endpoints + (discover_downloads, artist_bubbles, search_bubbles, beatport_bubbles), + which differ only by: + + - ``payload_key`` ('downloads' for discover, 'bubbles' for the rest) and + its ``no_data_error`` message. + - ``snapshot_kind`` — the db.save_bubble_snapshot category. + - ``success_noun`` — fills "Snapshot saved with N ". + - ``log_subject`` / ``log_noun`` — the info ("Saved : N ") + and except ("Error saving ") log lines. + + Returns ``(payload, status_code)``. ``get_json`` is invoked inside the try + like the original ``request.json``. + """ + try: + from datetime import datetime + + data = get_json() + if not data or payload_key not in data: + return {'success': False, 'error': no_data_error}, 400 + + items = data[payload_key] + + db = get_database() + db.save_bubble_snapshot(snapshot_kind, items, profile_id=get_current_profile_id()) + + count = len(items) + logger.info(f"Saved {log_subject}: {count} {log_noun}") + + return { + 'success': True, + 'message': f'Snapshot saved with {count} {success_noun}', + 'timestamp': datetime.now().isoformat(), + }, 200 + except Exception as e: + logger.error(f"Error saving {log_subject}: {e}") + import traceback + traceback.print_exc() + return {'success': False, 'error': str(e)}, 500 + + def update_playlist_phase( states: Dict[str, Any], key: str, diff --git a/tests/discovery/test_discovery_endpoints.py b/tests/discovery/test_discovery_endpoints.py index 618e18cc..2f1b0ace 100644 --- a/tests/discovery/test_discovery_endpoints.py +++ b/tests/discovery/test_discovery_endpoints.py @@ -877,3 +877,81 @@ def test_update_phase_exception_500(): error_label='Tidal', valid_phases=_PHASES, apply_extra_fields=False) assert code == 500 and 'error' in body + + +# --------------------------------------------------------------------------- +# save_bubble_snapshot +# --------------------------------------------------------------------------- + +class _SnapDB: + def __init__(self): + self.saved = [] + + def save_bubble_snapshot(self, kind, items, profile_id=None): + self.saved.append((kind, items, profile_id)) + + +def _snap_kwargs(db, *, payload_key='bubbles', no_data_error='No bubble data provided', + snapshot_kind='artist_bubbles', success_noun='artist bubbles', + log_subject='artist bubble snapshot', log_noun='artists'): + return dict( + payload_key=payload_key, no_data_error=no_data_error, snapshot_kind=snapshot_kind, + success_noun=success_noun, log_subject=log_subject, log_noun=log_noun, + get_database=lambda: db, get_current_profile_id=lambda: 7, + ) + + +def test_snapshot_missing_payload_key(): + from core.discovery.endpoints import save_bubble_snapshot + db = _SnapDB() + body, code = save_bubble_snapshot(lambda: {'other': 1}, **_snap_kwargs(db)) + assert code == 400 and body == {'success': False, 'error': 'No bubble data provided'} + assert db.saved == [] + + +def test_snapshot_none_body(): + from core.discovery.endpoints import save_bubble_snapshot + db = _SnapDB() + body, code = save_bubble_snapshot(lambda: None, + **_snap_kwargs(db, payload_key='downloads', + no_data_error='No download data provided')) + assert code == 400 and body['error'] == 'No download data provided' + + +def test_snapshot_happy_path(): + from core.discovery.endpoints import save_bubble_snapshot + db = _SnapDB() + items = [{'a': 1}, {'b': 2}, {'c': 3}] + body, code = save_bubble_snapshot(lambda: {'bubbles': items}, + **_snap_kwargs(db, snapshot_kind='search_bubbles', + success_noun='search bubbles')) + assert code == 200 + assert body['success'] is True + assert body['message'] == 'Snapshot saved with 3 search bubbles' + assert isinstance(body['timestamp'], str) and body['timestamp'] + # persisted with kind + profile id + assert db.saved == [('search_bubbles', items, 7)] + + +def test_snapshot_discover_downloads_key(): + from core.discovery.endpoints import save_bubble_snapshot + db = _SnapDB() + body, code = save_bubble_snapshot(lambda: {'downloads': [1, 2]}, + **_snap_kwargs(db, payload_key='downloads', + no_data_error='No download data provided', + snapshot_kind='discover_downloads', + success_noun='downloads', + log_subject='discover download snapshot', + log_noun='downloads')) + assert code == 200 + assert body['message'] == 'Snapshot saved with 2 downloads' + assert db.saved[0][0] == 'discover_downloads' + + +def test_snapshot_exception_returns_500(): + from core.discovery.endpoints import save_bubble_snapshot + class _BoomDB: + def save_bubble_snapshot(self, *a, **k): + raise RuntimeError('db down') + body, code = save_bubble_snapshot(lambda: {'bubbles': [1]}, **_snap_kwargs(_BoomDB())) + assert code == 500 and body == {'success': False, 'error': 'db down'} diff --git a/web_server.py b/web_server.py index a4cc81e5..17c4322e 100644 --- a/web_server.py +++ b/web_server.py @@ -20856,6 +20856,7 @@ from core.discovery.endpoints import ( start_sync as _start_sync_core, update_discovery_match as _update_discovery_match_core, update_playlist_phase as _update_playlist_phase_core, + save_bubble_snapshot as _save_bubble_snapshot_core, playlist_name_attr_or_unknown as _pl_name_attr_or_unknown, playlist_name_strict as _pl_name_strict, playlist_name_safe as _pl_name_safe, @@ -20985,6 +20986,19 @@ def _update_source_playlist_phase(states, key, not_found_message, error_label, return jsonify(body), code +def _save_source_bubble_snapshot(payload_key, no_data_error, snapshot_kind, + success_noun, log_subject, log_noun): + """Thin glue for the snapshot routes (discover_downloads / artist_bubbles / + search_bubbles / beatport_bubbles).""" + body, code = _save_bubble_snapshot_core( + lambda: request.json, payload_key=payload_key, no_data_error=no_data_error, + snapshot_kind=snapshot_kind, success_noun=success_noun, log_subject=log_subject, + log_noun=log_noun, get_database=get_database, + get_current_profile_id=get_current_profile_id, + ) + return jsonify(body), code + + def _build_tidal_discovery_deps(): """Build the TidalDiscoveryDeps bundle from web_server.py globals on each call.""" return _discovery_tidal.TidalDiscoveryDeps( @@ -23519,35 +23533,7 @@ def save_discover_download_snapshot(): """ Saves a snapshot of current discover download state for persistence across page refreshes. """ - try: - from datetime import datetime - - data = request.json - if not data or 'downloads' not in data: - return jsonify({'success': False, 'error': 'No download data provided'}), 400 - - downloads = data['downloads'] - - db = get_database() - db.save_bubble_snapshot('discover_downloads', downloads, profile_id=get_current_profile_id()) - - download_count = len(downloads) - logger.info(f"Saved discover download snapshot: {download_count} downloads") - - return jsonify({ - 'success': True, - 'message': f'Snapshot saved with {download_count} downloads', - 'timestamp': datetime.now().isoformat() - }) - - except Exception as e: - logger.error(f"Error saving discover download snapshot: {e}") - import traceback - traceback.print_exc() - return jsonify({ - 'success': False, - 'error': str(e) - }), 500 + return _save_source_bubble_snapshot("downloads", "No download data provided", "discover_downloads", "downloads", "discover download snapshot", "downloads") @app.route('/api/discover_downloads/hydrate', methods=['GET']) def hydrate_discover_downloads(): @@ -23668,35 +23654,7 @@ def save_artist_bubble_snapshot(): """ Saves a snapshot of current artist bubble state for persistence across page refreshes. """ - try: - from datetime import datetime - - data = request.json - if not data or 'bubbles' not in data: - return jsonify({'success': False, 'error': 'No bubble data provided'}), 400 - - bubbles = data['bubbles'] - - db = get_database() - db.save_bubble_snapshot('artist_bubbles', bubbles, profile_id=get_current_profile_id()) - - bubble_count = len(bubbles) - logger.info(f"Saved artist bubble snapshot: {bubble_count} artists") - - return jsonify({ - 'success': True, - 'message': f'Snapshot saved with {bubble_count} artist bubbles', - 'timestamp': datetime.now().isoformat() - }) - - except Exception as e: - logger.error(f"Error saving artist bubble snapshot: {e}") - import traceback - traceback.print_exc() - return jsonify({ - 'success': False, - 'error': str(e) - }), 500 + return _save_source_bubble_snapshot("bubbles", "No bubble data provided", "artist_bubbles", "artist bubbles", "artist bubble snapshot", "artists") @app.route('/api/artist_bubbles/hydrate', methods=['GET']) def hydrate_artist_bubbles(): @@ -23840,35 +23798,7 @@ def save_search_bubble_snapshot(): """ Saves a snapshot of current search bubble state for persistence across page refreshes. """ - try: - from datetime import datetime - - data = request.json - if not data or 'bubbles' not in data: - return jsonify({'success': False, 'error': 'No bubble data provided'}), 400 - - bubbles = data['bubbles'] - - db = get_database() - db.save_bubble_snapshot('search_bubbles', bubbles, profile_id=get_current_profile_id()) - - bubble_count = len(bubbles) - logger.info(f"Saved search bubble snapshot: {bubble_count} albums/tracks") - - return jsonify({ - 'success': True, - 'message': f'Snapshot saved with {bubble_count} search bubbles', - 'timestamp': datetime.now().isoformat() - }) - - except Exception as e: - logger.error(f"Error saving search bubble snapshot: {e}") - import traceback - traceback.print_exc() - return jsonify({ - 'success': False, - 'error': str(e) - }), 500 + return _save_source_bubble_snapshot("bubbles", "No bubble data provided", "search_bubbles", "search bubbles", "search bubble snapshot", "albums/tracks") @app.route('/api/search_bubbles/hydrate', methods=['GET']) def hydrate_search_bubbles(): @@ -24003,35 +23933,7 @@ def hydrate_search_bubbles(): @app.route('/api/beatport_bubbles/snapshot', methods=['POST']) def save_beatport_bubble_snapshot(): """Saves a snapshot of current Beatport download bubble state for persistence.""" - try: - from datetime import datetime - - data = request.json - if not data or 'bubbles' not in data: - return jsonify({'success': False, 'error': 'No bubble data provided'}), 400 - - bubbles = data['bubbles'] - - db = get_database() - db.save_bubble_snapshot('beatport_bubbles', bubbles, profile_id=get_current_profile_id()) - - bubble_count = len(bubbles) - logger.info(f"Saved Beatport bubble snapshot: {bubble_count} charts") - - return jsonify({ - 'success': True, - 'message': f'Snapshot saved with {bubble_count} Beatport bubbles', - 'timestamp': datetime.now().isoformat() - }) - - except Exception as e: - logger.error(f"Error saving Beatport bubble snapshot: {e}") - import traceback - traceback.print_exc() - return jsonify({ - 'success': False, - 'error': str(e) - }), 500 + return _save_source_bubble_snapshot("bubbles", "No bubble data provided", "beatport_bubbles", "Beatport bubbles", "Beatport bubble snapshot", "charts") @app.route('/api/beatport_bubbles/hydrate', methods=['GET']) def hydrate_beatport_bubbles(): From b8384beef9a274cb3b536e7d1180f6c319110b7d Mon Sep 17 00:00:00 2001 From: Tyler Richardson-LaPlume <170156756+IamGroot60@users.noreply.github.com> Date: Fri, 29 May 2026 00:49:02 -0400 Subject: [PATCH 20/52] =?UTF-8?q?Fix:=20Usenet=20bundle=20stuck=20at=2099%?= =?UTF-8?q?/100%=20=E2=80=94=20SAB=20reports=20post-processing=20in=20Hist?= =?UTF-8?q?ory=20as=20non-terminal=20(#721)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The earlier #721 fix tolerated a ~10s "completed but no save_path" window, but the real production stall sits upstream of that: SABnzbd removes a finished download from the queue and runs par2 verify / repair / unpack *in History*, exposing the live stage in the slot `status` ('Verifying' / 'Repairing' / 'Extracting' / 'Moving' / ...) with `storage` empty until the final move. `_parse_history_slot` mapped EVERY non-'Failed' status to 'completed', so a still-extracting 1.7 GB FLAC album looked "completed with no save_path" the instant download hit 100%. The poll burned its completed-no-path budget mid-PP and bailed, freezing the UI on the last download emit (the stuck-at-99%/100% signature). SAB then finished fine — which is why the job shows Completed in History but SoulSync never staged it. Root fix - `_parse_history_slot` routes `status` through `_map_state`, so PP stages stay NON-terminal: the poll keeps waiting (as 'downloading') for as long as post-processing takes and only a real 'Completed' flips to terminal success. `save_path` is trusted only on true completion (mid-PP path fields may point at the incomplete dir). Supporting / defensive - `UsenetStatus.incomplete_path`: surfaced separately from save_path (SAB `incomplete_path`) and used by the poll loops as a LAST RESORT after the completed-no-path window, to recover the case where `storage` never lands but the files are physically on disk. - `poll_album_download`: dedicated, configurable completed-no-path window (~120s via `download_source.album_bundle_completed_no_path_seconds`) decoupled from the ~10s transient-miss window; incomplete_path fallback; a 30s heartbeat log so the previously-silent poll loop is diagnosable. - `usenet.py` `_download_thread`: per-track parity — it was erroring immediately on the first completed-no-path read. - `album_bundle_dispatch.py` / `status.py` / `monitor.py`: use the project `get_logger` so download-flow logs land in app.log under the `soulsync.*` namespace (they were console-only before, which hid the `[Album Bundle] flow failed` line during triage). Tests - PP-history state mapping; end-to-end Hunky Dory PP regression (download -> Verifying/Extracting in History past both budgets -> Completed+storage -> success); completed-no-path window + incomplete_path fallback; per-track thread parity. ruff + compileall + pytest all green (the only local failures are environmental: missing tzdata + local tools/ffmpeg.exe, neither present on CI). Co-Authored-By: Claude Opus 4.8 (1M context) --- core/download_plugins/album_bundle.py | 120 ++++++++++++++-- core/download_plugins/usenet.py | 47 ++++++- core/downloads/album_bundle_dispatch.py | 11 +- core/downloads/monitor.py | 5 +- core/downloads/status.py | 5 +- core/usenet_clients/base.py | 9 ++ core/usenet_clients/sabnzbd.py | 63 +++++++-- tests/test_album_bundle.py | 101 ++++++++++++++ tests/test_torrent_usenet_plugins.py | 96 +++++++++++++ tests/test_usenet_client_adapters.py | 177 ++++++++++++++++++++++++ 10 files changed, 605 insertions(+), 29 deletions(-) diff --git a/core/download_plugins/album_bundle.py b/core/download_plugins/album_bundle.py index 746022a0..6abe6328 100644 --- a/core/download_plugins/album_bundle.py +++ b/core/download_plugins/album_bundle.py @@ -202,6 +202,34 @@ def get_transient_miss_threshold() -> int: return DEFAULT_TRANSIENT_MISS_THRESHOLD +# How long to keep polling after the client reports terminal success +# but hasn't yet exposed a final save_path. Distinct from the +# transient-miss threshold because the two model different things: +# a transient miss is "the job vanished — fail fast (~10s) so a deleted +# job doesn't hang"; a completed-no-path read is "the download SUCCEEDED +# and the files are on disk — SAB just hasn't finished writing the +# ``storage`` field." The #706 fix reused the 5-poll (~10s) miss window +# here, but #721's own report shows SAB can take 2+ minutes (or, on some +# versions, never expose ``storage`` at all) — so a 10s window false-fails +# a download that actually completed. Expressed in SECONDS (converted to +# a poll count against the live interval) so it's interval-independent. +# Override via ``download_source.album_bundle_completed_no_path_seconds``. +DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS = 120.0 + + +def get_completed_no_path_window_seconds() -> float: + """Return the completed-but-no-save_path tolerance window (seconds).""" + raw = config_manager.get('download_source.album_bundle_completed_no_path_seconds', + DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS) + try: + value = float(raw) + if value > 0: + return value + except (TypeError, ValueError): + pass + return DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS + + class TransientMissCounter: """Bounded retry counter for adapter status reads. @@ -237,6 +265,7 @@ def poll_album_download( failed_states: frozenset = frozenset(['failed']), is_shutdown: Optional[Callable[[], bool]] = None, transient_miss_threshold: int = DEFAULT_TRANSIENT_MISS_THRESHOLD, + completed_no_path_threshold: Optional[int] = None, poll_interval: Optional[float] = None, timeout: Optional[float] = None, sleep: Callable[[float], None] = time.sleep, @@ -270,16 +299,28 @@ def poll_album_download( 'error' → poll infinite-looped until the 6-hour timeout. - ``transient_miss_threshold`` is the number of consecutive None / 'error' reads tolerated before declaring the job gone. Sized for - the SAB queue→history gap window. + the SAB queue→history gap window (~10s) — a vanished job should + fail fast. + - ``completed_no_path_threshold`` is a SEPARATE, longer window for + the "client says complete but no save_path yet" case. The download + already succeeded, so this defaults to ~120s (configurable via + ``download_source.album_bundle_completed_no_path_seconds``) instead + of reusing the 10s miss window — #721 showed SAB can take 2+ minutes + to write ``storage``. When the window is exhausted the loop falls + back to the adapter's ``incomplete_path`` (the on-disk in-progress + dir) if present, and only emits terminal ``failed`` when there's no + path of any kind to scan. - Returns the adapter's reported save_path on terminal success, or - ``None`` on any failure (timeout / disappeared / explicit failed - / shutdown). On every failure path emits ``'failed'`` once with an - ``error`` field describing why. + Returns the adapter's reported save_path (or, as a last resort, its + ``incomplete_path``) on terminal success, or ``None`` on any failure + (timeout / disappeared / explicit failed / shutdown). On every + failure path emits ``'failed'`` once with an ``error`` field + describing why. """ interval = poll_interval if poll_interval is not None else get_poll_interval() deadline = monotonic() + (timeout if timeout is not None else get_poll_timeout()) last_save_path: Optional[str] = None + last_incomplete_path: Optional[str] = None misses = TransientMissCounter(transient_miss_threshold) # Separate counter for "client reports terminal-success state but no # save_path field has landed yet." SAB History flips ``status`` to @@ -290,9 +331,19 @@ def poll_album_download( # seconds because ``storage`` isn't populated yet. Pre-fix the # poll returned ``None`` on the first such read, the bundle # plugin marked the batch failed, but the UI still displayed the - # last ``downloading`` progress emit. Now we retry up to the - # same threshold so SAB has a window to write the path. - completed_no_path_misses = TransientMissCounter(transient_miss_threshold) + # last ``downloading`` progress emit. + # + # This window is intentionally LONGER than the transient-miss window: + # the download already SUCCEEDED, so being patient here is cheap and + # correct, whereas the original 5-poll (~10s) reuse false-failed real + # completions (#721 reported SAB taking 2+ minutes). Default ~120s, + # converted from seconds to a poll count against the live interval. + if completed_no_path_threshold is None: + completed_no_path_threshold = max( + transient_miss_threshold, + int(get_completed_no_path_window_seconds() / max(interval, 0.001)) or 1, + ) + completed_no_path_misses = TransientMissCounter(completed_no_path_threshold) def _fail(reason: str) -> None: try: @@ -300,6 +351,16 @@ def poll_album_download( except Exception as cb_exc: logger.debug("%s terminal emit failed: %s", log_prefix, cb_exc) + # Heartbeat so the otherwise-silent download loop is diagnosable. + # The loop emits progress to the UI on every poll but logs nothing + # during normal operation — which made the #721 "stuck at N%" reports + # impossible to triage from logs alone (we couldn't tell if the poll + # was alive, what state SAB returned, or whether it had wedged). Log + # the raw adapter read at most once per heartbeat interval. + HEARTBEAT_SECONDS = 30.0 + last_heartbeat = monotonic() + poll_count = 0 + while monotonic() < deadline: if is_shutdown and is_shutdown(): # Shutdown is a clean exit — don't paint failure on the UI; @@ -312,6 +373,21 @@ def poll_album_download( logger.warning("%s Poll error: %s", log_prefix, e) status = None + poll_count += 1 + now = monotonic() + if now - last_heartbeat >= HEARTBEAT_SECONDS: + last_heartbeat = now + if status is None: + logger.info("%s '%s' poll #%d: client returned no status (miss %d/%d)", + log_prefix, title, poll_count, misses.misses, misses.threshold) + else: + logger.info( + "%s '%s' poll #%d: state=%r progress=%.2f save_path=%r", + log_prefix, title, poll_count, + getattr(status, 'state', None), getattr(status, 'progress', 0.0) or 0.0, + getattr(status, 'save_path', None), + ) + if status is None: if misses.record_miss(): logger.error( @@ -334,6 +410,12 @@ def poll_album_download( speed=status.download_speed) if status.save_path: last_save_path = status.save_path + # Remember the in-progress dir too — never used on a normal + # completion, only as the last-resort fallback below when the + # final save_path provably never lands. + incomplete_path = getattr(status, 'incomplete_path', None) + if incomplete_path: + last_incomplete_path = incomplete_path if status.state in complete_states: if last_save_path: @@ -341,11 +423,27 @@ def poll_album_download( return last_save_path # Terminal-success state but no save_path landed yet. # SAB History flips ``Completed`` a few seconds before - # ``storage`` is populated — give the adapter a few more - # polls before declaring this a hard failure. Without this + # ``storage`` is populated — give the adapter a generous + # window before declaring this a hard failure. Without this # tolerance, every TAR / unrar-bearing usenet release # would race the path-write window and randomly fail. if completed_no_path_misses.record_miss(): + # Last resort before failing: SAB finished and the files + # are physically on disk (#721), but the final ``storage`` + # field never landed. Fall back to the in-progress dir so + # the bundle can still scan + stage the audio, rather than + # leaving the user stuck with a completed-in-SAB download + # that SoulSync never imports. + if last_incomplete_path: + logger.warning( + "%s '%s' completed on the client but never exposed a final " + "save_path after %d polls — falling back to the in-progress " + "path %r as a last resort. If staging fails, the SAB job " + "likely needs its post-process move to finish first.", + log_prefix, title, completed_no_path_misses.misses, + last_incomplete_path, + ) + return last_incomplete_path logger.error( "%s '%s' reported terminal success but no save_path landed " "after %d consecutive polls — bundle cannot stage. Adapter " @@ -419,9 +517,11 @@ __all__ = [ "DEFAULT_POLL_INTERVAL_SECONDS", "DEFAULT_POLL_TIMEOUT_SECONDS", "DEFAULT_TRANSIENT_MISS_THRESHOLD", + "DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS", "TransientMissCounter", "atomic_copy_to_staging", "copy_audio_files_atomically", + "get_completed_no_path_window_seconds", "get_poll_interval", "get_poll_timeout", "get_transient_miss_threshold", diff --git a/core/download_plugins/usenet.py b/core/download_plugins/usenet.py index 8988fec4..75302c50 100644 --- a/core/download_plugins/usenet.py +++ b/core/download_plugins/usenet.py @@ -24,6 +24,7 @@ from core.archive_pipeline import collect_audio_after_extraction from core.download_plugins.album_bundle import ( TransientMissCounter, copy_audio_files_atomically, + get_completed_no_path_window_seconds, pick_best_album_release, poll_album_download, ) @@ -229,11 +230,22 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS last_save_path: Optional[str] = None + last_incomplete_path: Optional[str] = None # Tolerate transient None / unmapped 'error' reads — SAB # removes a job from the queue before adding it to history, # and on busy servers that gap spans several polls. See # ``album_bundle.TransientMissCounter`` for the shared rule. misses = TransientMissCounter() + # Separate, LONGER window for "SAB says completed but hasn't + # written the final save_path yet" — the per-track sibling of the + # bundle fix (#721). Without this the thread called + # ``_finalize_download(None)`` on the first Completed-no-path read + # and errored a download that actually succeeded in SAB. Default + # ~120s, converted to a poll count against the live interval. + completed_no_path_misses = TransientMissCounter( + max(misses.threshold, + int(get_completed_no_path_window_seconds() / max(_POLL_INTERVAL_SECONDS, 0.001)) or 1) + ) while time.monotonic() < deadline: if self.shutdown_check and self.shutdown_check(): return @@ -267,10 +279,41 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): row['error'] = status.error if status.save_path: last_save_path = status.save_path + incomplete_path = getattr(status, 'incomplete_path', None) + if incomplete_path: + last_incomplete_path = incomplete_path if status.state in _COMPLETE_STATES: - self._finalize_download(download_id, last_save_path) - return + if last_save_path: + self._finalize_download(download_id, last_save_path) + return + # Completed but no final save_path yet — SAB flips + # History to 'Completed' before writing ``storage``. + # Wait out the (longer) completed-no-path window rather + # than erroring a download that actually succeeded. + if completed_no_path_misses.record_miss(): + if last_incomplete_path: + logger.warning( + "Usenet %s: '%s' completed but no final save_path after " + "%d polls — falling back to in-progress path %r", + download_id[:8], job_id, completed_no_path_misses.misses, + last_incomplete_path, + ) + self._finalize_download(download_id, last_incomplete_path) + return + self._mark_error( + download_id, + "Usenet job completed but client never reported a save_path", + ) + return + logger.info( + "Usenet %s: '%s' completed on client but save_path not yet set — " + "retrying (poll %d/%d)", + download_id[:8], job_id, + completed_no_path_misses.misses, completed_no_path_misses.threshold, + ) + time.sleep(_POLL_INTERVAL_SECONDS) + continue if status.state == 'failed': self._mark_error(download_id, status.error or "Usenet client reported failure") return diff --git a/core/downloads/album_bundle_dispatch.py b/core/downloads/album_bundle_dispatch.py index 734d35a9..634c0e95 100644 --- a/core/downloads/album_bundle_dispatch.py +++ b/core/downloads/album_bundle_dispatch.py @@ -31,11 +31,18 @@ testable without touching live runtime state. from __future__ import annotations -import logging from pathlib import Path from typing import Any, Callable, Optional, Protocol -logger = logging.getLogger(__name__) +from utils.logging_config import get_logger + +# Use the project logger factory so these lines land in app.log under the +# ``soulsync.*`` namespace the file handler captures. Plain +# ``logging.getLogger(__name__)`` logs to the console only (the file +# handler is attached to the ``soulsync`` logger), which is why +# ``[Album Bundle] flow failed`` showed up in the terminal but never in +# app.log during the #721 triage. +logger = get_logger("downloads.album_bundle_dispatch") class BatchStateAccess(Protocol): diff --git a/core/downloads/monitor.py b/core/downloads/monitor.py index c4b645f5..4bcb0bc3 100644 --- a/core/downloads/monitor.py +++ b/core/downloads/monitor.py @@ -5,7 +5,6 @@ The class body is byte-identical to the original. Module-level globals helpers and orchestrator handles. ``IS_SHUTTING_DOWN`` is a module-level flag mirrored from web_server's own flag in ``_shutdown_runtime_components``. """ -import logging import threading import time @@ -18,8 +17,10 @@ from core.runtime_state import ( tasks_lock, ) from utils.async_helpers import run_async +from utils.logging_config import get_logger -logger = logging.getLogger(__name__) +# Project logger factory so these lines reach app.log (soulsync.* namespace). +logger = get_logger("downloads.monitor") # Mirrored from web_server.IS_SHUTTING_DOWN via _shutdown_runtime_components. IS_SHUTTING_DOWN = False diff --git a/core/downloads/status.py b/core/downloads/status.py index 2d2073db..010581b9 100644 --- a/core/downloads/status.py +++ b/core/downloads/status.py @@ -20,7 +20,6 @@ are passed via `StatusDeps` so the module is web_server-import-free. from __future__ import annotations -import logging import threading import time from dataclasses import dataclass @@ -32,8 +31,10 @@ from core.runtime_state import ( download_tasks, tasks_lock, ) +from utils.logging_config import get_logger -logger = logging.getLogger(__name__) +# Project logger factory so these lines reach app.log (soulsync.* namespace). +logger = get_logger("downloads.status") def _schedule_completion_callback(deps, batch_id: str, task_id: str, success: bool) -> None: diff --git a/core/usenet_clients/base.py b/core/usenet_clients/base.py index f77b3d78..12b25578 100644 --- a/core/usenet_clients/base.py +++ b/core/usenet_clients/base.py @@ -39,6 +39,15 @@ class UsenetStatus: download_speed: int # bytes/sec eta: Optional[int] = None # seconds, None if unknown save_path: Optional[str] = None + # In-progress / pre-move directory (SAB ``incomplete_path``). Kept + # SEPARATE from ``save_path`` on purpose: it points at the staging + # dir SAB uses BEFORE its post-process move, so it must never be + # treated as the final path on a normal completion. The poll loops + # only fall back to it as a LAST RESORT — after waiting the full + # completed-but-no-save_path window — to recover the (#721) case + # where SAB finished, the files are physically on disk, but the + # final ``storage`` field never lands. See ``poll_album_download``. + incomplete_path: Optional[str] = None category: Optional[str] = None files: Optional[List[str]] = None error: Optional[str] = None diff --git a/core/usenet_clients/sabnzbd.py b/core/usenet_clients/sabnzbd.py index e6c30a9f..daf55d65 100644 --- a/core/usenet_clients/sabnzbd.py +++ b/core/usenet_clients/sabnzbd.py @@ -232,19 +232,54 @@ class SABnzbdAdapter: ) def _parse_history_slot(self, slot: dict) -> UsenetStatus: - # History entries are post-download — progress is 1.0 unless failed. - sab_state = (slot.get('status') or '').lower() - is_failed = sab_state == 'failed' - save_path = self._extract_history_save_path(slot) + # History entries cover BOTH finished jobs AND jobs that are still + # POST-PROCESSING. SAB removes a job from the queue the moment the + # download finishes and runs par2 verify / repair / unpack / move + # while the job sits in History — exposing the live stage in the + # ``status`` field ('Verifying' / 'Repairing' / 'Extracting' / + # 'Moving' / 'Running' / ...). Only 'Completed' means truly done; + # 'Failed' / 'Deleted' mean failure. + # + # The old logic mapped EVERY non-'failed' status to 'completed'. + # That made the poll treat a still-extracting 1.7 GB FLAC album + # (status 'Extracting', ``storage`` not written yet) as "completed + # but no save_path" and burn the completed-no-path window mid-PP — + # exactly the #721 stuck-at-99% signature in production where the + # path IS shared. Route the status through the same queue-state map + # so PP stages stay NON-terminal: the poll keeps waiting (as + # 'downloading') for as long as post-processing takes, and only a + # real 'Completed' flips it to terminal success. + mapped = _map_state(slot.get('status') or '') + is_failed = mapped == 'failed' + is_completed = mapped == 'completed' + # Only trust the final path on a TRUE completion. Mid-PP the path + # fields may be empty or still point at the incomplete dir; the + # completed-no-path retry handles the brief gap between the + # 'Completed' flip and ``storage`` landing in the same response. + save_path = self._extract_history_save_path(slot) if is_completed else None + # ``incomplete_path`` is surfaced separately (NOT as save_path) so + # the poll loops can fall back to it only as a last resort once + # the final ``storage`` field has provably failed to land — see + # ``UsenetStatus.incomplete_path`` and the #721 fallback in + # ``poll_album_download``. Whitespace-only values are treated as + # absent, same as the save_path chain. + incomplete_path = slot.get('incomplete_path') + if not (isinstance(incomplete_path, str) and incomplete_path.strip()): + incomplete_path = None + bytes_total = int(slot.get('bytes') or 0) return UsenetStatus( id=str(slot.get('nzo_id') or ''), name=slot.get('name') or '', - state='failed' if is_failed else 'completed', + state=mapped, + # Download itself is finished for ANY History slot (in-PP or + # done), so report full download progress — don't snap the UI + # back to 0% while SAB verifies/unpacks. Failed = 0. progress=0.0 if is_failed else 1.0, - size=int(slot.get('bytes') or 0), - downloaded=int(slot.get('bytes') or 0) if not is_failed else 0, + size=bytes_total, + downloaded=0 if is_failed else bytes_total, download_speed=0, save_path=save_path, + incomplete_path=incomplete_path, category=slot.get('category'), error=slot.get('fail_message') if is_failed else None, ) @@ -279,10 +314,16 @@ class SABnzbdAdapter: value = slot.get(key) if value and isinstance(value, str) and value.strip(): return value - # Loud diagnostic when the bundle poll is about to fail on this: - # users on SAB versions / forks with novel field names need to - # see this in the logs so we can grow ``_HISTORY_SAVE_PATH_KEYS``. - logger.debug( + # Loud diagnostic when the bundle poll is about to wait/fail on + # this: users on SAB versions / forks with novel field names need + # to see this in the logs so we can grow ``_HISTORY_SAVE_PATH_KEYS``. + # INFO (not DEBUG) on purpose — a completed History slot with no + # resolvable path is the #721 stuck-bundle signature, and dumping + # the actual slot keys here is what lets us add the missing field + # without a debug-logging round-trip. It only fires for completed + # slots that lack every known path field, so it self-limits the + # moment ``storage`` lands. + logger.info( "[SAB] History slot for nzo_id=%s has no save_path in any " "of the known fields %r — slot keys: %r", slot.get('nzo_id'), diff --git a/tests/test_album_bundle.py b/tests/test_album_bundle.py index a628b68b..57790286 100644 --- a/tests/test_album_bundle.py +++ b/tests/test_album_bundle.py @@ -22,10 +22,12 @@ import pytest from core.download_plugins.album_bundle import ( ALBUM_PICK_MAX_BYTES, ALBUM_PICK_MIN_BYTES, + DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS, DEFAULT_POLL_INTERVAL_SECONDS, DEFAULT_POLL_TIMEOUT_SECONDS, atomic_copy_to_staging, copy_audio_files_atomically, + get_completed_no_path_window_seconds, get_poll_interval, get_poll_timeout, pick_best_album_release, @@ -301,6 +303,28 @@ def test_get_poll_timeout_falls_back_on_garbage() -> None: assert get_poll_timeout() == DEFAULT_POLL_TIMEOUT_SECONDS +def test_get_completed_no_path_window_uses_default_when_unset() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS + assert get_completed_no_path_window_seconds() == DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS + + +def test_get_completed_no_path_window_honours_override() -> None: + """Users whose SAB is slow to write ``storage`` (large box sets, + slow disks) can widen the tolerance without touching code.""" + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = 300 + assert get_completed_no_path_window_seconds() == 300.0 + + +def test_get_completed_no_path_window_falls_back_on_garbage() -> None: + with patch('core.download_plugins.album_bundle.config_manager') as cm: + cm.get.return_value = '' + assert get_completed_no_path_window_seconds() == DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS + cm.get.return_value = 0 + assert get_completed_no_path_window_seconds() == DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS + + # --------------------------------------------------------------------------- # poll_album_download — lifted poll loop for both torrent + usenet plugins. # --------------------------------------------------------------------------- @@ -395,6 +419,7 @@ class _Status: downloaded: int = 0 download_speed: int = 0 error: Optional[str] = None + incomplete_path: Optional[str] = None class _ScriptedClock: @@ -657,6 +682,82 @@ def test_poll_gives_up_when_completed_with_no_save_path_persists() -> None: assert 'save_path' in err or 'success but never' in err +def test_poll_completed_no_path_window_is_longer_than_miss_window() -> None: + """#721 follow-up: the completed-but-no-save_path window must be + DECOUPLED from (and far longer than) the transient-miss window. SAB + can take 2+ minutes to write ``storage``; the old code reused the + 5-poll (~10s) miss window here and false-failed real completions. + With a small miss threshold but the default long no-path window, a + download that takes 8 completed-no-path polls before ``storage`` + lands must still succeed.""" + clock = _ScriptedClock() + emit, calls = _make_emit_recorder() + sequence = iter( + [_Status(state='completed', save_path=None, progress=1.0)] * 8 + + [_Status(state='completed', save_path='/dl/late', progress=1.0)] + ) + result = poll_album_download( + get_status=lambda: next(sequence), + title='Slow SAB', + emit=emit, + complete_states=frozenset(['completed']), + transient_miss_threshold=3, # vanished-job window stays short + # completed_no_path_threshold left to default (~120s / interval). + sleep=clock.sleep, monotonic=clock.monotonic, + poll_interval=2.0, timeout=600.0, + ) + assert result == '/dl/late' + assert 'failed' not in [c[0] for c in calls] + + +def test_poll_falls_back_to_incomplete_path_after_window_exhausted() -> None: + """When SAB reports the job completed but the final save_path NEVER + lands (some SAB versions / no post-process move), the files are + still physically on disk in the in-progress dir. Rather than failing + a download that actually succeeded, the poll falls back to the + adapter's ``incomplete_path`` as a last resort once the window is + exhausted — no terminal 'failed' emit.""" + clock = _ScriptedClock() + emit, calls = _make_emit_recorder() + result = poll_album_download( + get_status=lambda: _Status( + state='completed', save_path=None, + incomplete_path='/sab/incomplete/album', progress=1.0, + ), + title='No Storage Field', + emit=emit, + complete_states=frozenset(['completed']), + completed_no_path_threshold=3, + sleep=clock.sleep, monotonic=clock.monotonic, + poll_interval=2.0, timeout=600.0, + ) + assert result == '/sab/incomplete/album' + assert 'failed' not in [c[0] for c in calls] + + +def test_poll_fails_when_no_path_and_no_incomplete_path() -> None: + """Last resort only fires when there's actually a path to scan. + With neither a final save_path nor an incomplete_path, the poll + still fails loudly after the window so the UI doesn't freeze.""" + clock = _ScriptedClock() + emit, calls = _make_emit_recorder() + result = poll_album_download( + get_status=lambda: _Status(state='completed', save_path=None, + incomplete_path=None, progress=1.0), + title='Truly Pathless', + emit=emit, + complete_states=frozenset(['completed']), + completed_no_path_threshold=3, + sleep=clock.sleep, monotonic=clock.monotonic, + poll_interval=2.0, timeout=600.0, + ) + assert result is None + failed_calls = [c for c in calls if c[0] == 'failed'] + assert len(failed_calls) == 1 + err = failed_calls[0][1].get('error', '').lower() + assert 'save_path' in err or 'success but never' in err + + def test_poll_uses_save_path_from_earlier_downloading_emit_if_completed_lacks_one() -> None: """Sticky save_path: when an earlier ``downloading`` status carried a non-empty ``save_path`` (qBit shows the target dir mid-download), diff --git a/tests/test_torrent_usenet_plugins.py b/tests/test_torrent_usenet_plugins.py index 1dd9f75b..8b53c6f9 100644 --- a/tests/test_torrent_usenet_plugins.py +++ b/tests/test_torrent_usenet_plugins.py @@ -361,6 +361,102 @@ def test_usenet_finalize_picks_first_audio_file(tmp_path: Path) -> None: assert plugin.active_downloads['u-1']['file_path'].endswith('track1.flac') +class _FakeClock: + """Deterministic monotonic + sleep so the per-track poll loop runs + in microseconds and never actually blocks.""" + + def __init__(self) -> None: + self.now = 0.0 + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.now += seconds + + +def _drive_download_thread(plugin, statuses, *, window_seconds=10.0): + """Run ``_download_thread`` end-to-end against a scripted adapter. + + ``statuses`` is the sequence of ``UsenetStatus`` reads the poll loop + will see (one per poll). Returns the finished active_downloads row.""" + download_id = 'u-poll' + plugin.active_downloads[download_id] = { + 'id': download_id, 'filename': 'x', 'username': 'usenet', + 'display_name': 'X', 'state': 'Initializing', 'progress': 0.0, + 'size': 0, 'transferred': 0, 'speed': 0, 'file_path': None, + 'audio_files': [], 'job_id': None, 'error': None, + } + adapter = MagicMock() + adapter.is_configured.return_value = True + adapter.add_nzb.return_value = 'job1' + adapter.get_status.side_effect = list(statuses) + clock = _FakeClock() + with patch('core.download_plugins.usenet.get_active_usenet_adapter', return_value=adapter), \ + patch('core.download_plugins.usenet.run_async', side_effect=lambda x: x), \ + patch('core.download_plugins.usenet.get_completed_no_path_window_seconds', + return_value=window_seconds), \ + patch('core.download_plugins.usenet.time', clock), \ + patch('core.download_plugins.usenet.collect_audio_after_extraction', + return_value=[Path('/done/track1.flac')]): + plugin._download_thread(download_id, 'http://x/y.nzb') + return plugin.active_downloads[download_id] + + +def test_usenet_thread_waits_out_completed_no_path_then_finalizes(tmp_path: Path) -> None: + """Per-track sibling of the #721 bundle fix. SAB flips History to + 'completed' before writing ``storage`` — the thread must NOT error + on the first such read. It waits out the completed-no-path window; + when the path lands it finalizes as Succeeded.""" + plugin = UsenetDownloadPlugin() + statuses = [ + UsenetStatus(id='job1', name='A', state='downloading', progress=0.6, + size=100, downloaded=60, download_speed=10), + UsenetStatus(id='job1', name='A', state='completed', progress=1.0, + size=100, downloaded=100, download_speed=0, save_path=None), + UsenetStatus(id='job1', name='A', state='completed', progress=1.0, + size=100, downloaded=100, download_speed=0, save_path=None), + UsenetStatus(id='job1', name='A', state='completed', progress=1.0, + size=100, downloaded=100, download_speed=0, + save_path='/done/album'), + ] + row = _drive_download_thread(plugin, statuses) + assert row['state'] == 'Completed, Succeeded' + assert row['progress'] == 100.0 + assert row['file_path'] == str(Path('/done/track1.flac')) + + +def test_usenet_thread_falls_back_to_incomplete_path_when_storage_never_lands() -> None: + """If ``storage`` never lands but SAB exposed an ``incomplete_path`` + (files physically on disk), the thread recovers via the in-progress + dir as a last resort rather than erroring a completed download.""" + plugin = UsenetDownloadPlugin() + completed_no_path = UsenetStatus( + id='job1', name='A', state='completed', progress=1.0, + size=100, downloaded=100, download_speed=0, + save_path=None, incomplete_path='/sab/incomplete/A', + ) + # Window of 10s / 2s interval = 5 polls, floored at the miss + # threshold; supply plenty so the fallback fires. + row = _drive_download_thread(plugin, [completed_no_path] * 12) + assert row['state'] == 'Completed, Succeeded' + assert row['audio_files'] == [str(Path('/done/track1.flac'))] + + +def test_usenet_thread_errors_when_completed_with_no_path_at_all() -> None: + """No final save_path AND no incomplete_path → there's nothing to + scan, so the thread errors (rather than spinning or finalizing a + phantom path).""" + plugin = UsenetDownloadPlugin() + completed_no_path = UsenetStatus( + id='job1', name='A', state='completed', progress=1.0, + size=100, downloaded=100, download_speed=0, save_path=None, + ) + row = _drive_download_thread(plugin, [completed_no_path] * 12) + assert row['state'] == 'Completed, Errored' + assert 'save_path' in (row['error'] or '').lower() + + def test_usenet_is_configured_requires_both_sides() -> None: plugin = UsenetDownloadPlugin() fake_adapter = MagicMock() diff --git a/tests/test_usenet_client_adapters.py b/tests/test_usenet_client_adapters.py index 8e1c6d0b..1445a36c 100644 --- a/tests/test_usenet_client_adapters.py +++ b/tests/test_usenet_client_adapters.py @@ -254,6 +254,103 @@ def test_sab_history_save_path_ignores_incomplete_path() -> None: assert slot.save_path is None +def test_sab_history_post_processing_states_are_not_terminal() -> None: + """#721 production root cause: SAB keeps a job in History while it + post-processes (verify / repair / unpack / move), exposing the live + stage in ``status``. Those must map to NON-terminal states so the + poll keeps waiting — NOT to 'completed', which would make the poll + treat a still-extracting album as "completed but no save_path" and + bail mid-PP (the stuck-at-99% bug). Only a real 'Completed' is + terminal success.""" + adapter = _sab_with_config() + for sab_status, expected in [ + ('Verifying', 'verifying'), + ('Repairing', 'repairing'), + ('Extracting', 'extracting'), + ('Moving', 'extracting'), + ('Running', 'extracting'), + ('Queued', 'queued'), + ]: + slot = adapter._parse_history_slot({ + 'nzo_id': 'pp', 'name': 'PP', 'status': sab_status, + 'bytes': 1000, + # ``storage`` not written yet — PP still in flight. + }) + assert slot.state == expected, f'{sab_status} -> {slot.state}, want {expected}' + # Non-terminal: not 'completed', not 'failed'. + assert slot.state not in ('completed', 'failed') + # Download is done, so progress is full, but no final path yet. + assert slot.progress == 1.0 + assert slot.save_path is None + + +def test_sab_history_completed_still_resolves_save_path() -> None: + """The true-completion path is unchanged: status 'Completed' with a + populated ``storage`` resolves to terminal success + final path.""" + adapter = _sab_with_config() + slot = adapter._parse_history_slot({ + 'nzo_id': 'done', 'name': 'D', 'status': 'Completed', + 'bytes': 1000, 'storage': '/data/downloads/music/Album', + }) + assert slot.state == 'completed' + assert slot.progress == 1.0 + assert slot.save_path == '/data/downloads/music/Album' + + +def test_sab_history_post_processing_ignores_storage_until_completed() -> None: + """Even if SAB has written a (possibly incomplete-dir) path field + while still post-processing, the adapter must NOT expose it as + save_path until the slot flips to 'Completed' — otherwise the bundle + could stage from a half-unpacked directory.""" + adapter = _sab_with_config() + slot = adapter._parse_history_slot({ + 'nzo_id': 'pp2', 'name': 'PP2', 'status': 'Extracting', + 'bytes': 1000, 'storage': '/data/downloads/music/Album', + }) + assert slot.state == 'extracting' + assert slot.save_path is None + + +def test_sab_history_surfaces_incomplete_path_separately_from_save_path() -> None: + """``incomplete_path`` must be exposed on its OWN field, never folded + into ``save_path``. The poll loops fall back to it only as a last + resort after the completed-no-path window is exhausted (#721) — using + it as save_path would bypass that window and point staging at the + pre-move dir on every normal completion.""" + adapter = _sab_with_config() + slot = adapter._parse_history_slot({ + 'nzo_id': 'gap2', 'name': 'No Storage', 'status': 'Completed', + 'bytes': 0, + # storage / path / download_path / dirname all absent — the + # #721 gap window — but the in-progress dir is known. + 'incomplete_path': '/sab/incomplete/No Storage', + }) + assert slot.save_path is None + assert slot.incomplete_path == '/sab/incomplete/No Storage' + + +def test_sab_history_incomplete_path_ignores_whitespace_only() -> None: + adapter = _sab_with_config() + slot = adapter._parse_history_slot({ + 'nzo_id': 'gap3', 'name': 'WS', 'status': 'Completed', + 'bytes': 0, 'incomplete_path': ' ', + }) + assert slot.incomplete_path is None + + +def test_sab_history_prefers_storage_and_still_carries_incomplete_path() -> None: + """When ``storage`` IS present, save_path resolves to it normally, + and incomplete_path is carried alongside (harmless — the poll only + consults it when save_path never lands).""" + adapter = _sab_with_config() + slot = adapter._parse_history_slot({ + 'nzo_id': 'both', 'name': 'B', 'status': 'Completed', + 'bytes': 0, 'storage': '/final', 'incomplete_path': '/inc', + }) + assert slot.save_path == '/final' + assert slot.incomplete_path == '/inc' + + def test_sab_add_nzb_via_url_returns_first_nzo_id() -> None: adapter = _sab_with_config() with patch('core.usenet_clients.sabnzbd.http_requests.get', @@ -425,6 +522,86 @@ def test_sab_poll_recovers_after_queue_to_history_handoff_gap() -> None: assert 'failed' not in [e[0] for e in emits] +def test_sab_poll_waits_through_history_post_processing_then_completes() -> None: + """Integration regression for the #721 PRODUCTION root cause + (David Bowie - Hunky Dory, 1.7 GB FLAC, stuck at 99%). + + SAB moves a finished download into History and runs par2 verify + + unpack THERE, exposing the live stage in ``status`` while ``storage`` + stays empty until the final move. For a 1.7 GB FLAC album that + post-processing window is far longer than the ~10s completed-no-path + budget. Pre-fix the adapter mapped 'Extracting' → 'completed', so the + poll saw "completed but no save_path" the instant download finished, + burned its budget mid-PP, and bailed — freezing the UI on the last + 'downloading' emit (99%). SAB then finished fine, which is why the + job shows Completed in History but SoulSync never staged it. + + Post-fix the in-PP History slots map to NON-terminal states, so the + poll just keeps waiting (as 'downloading') until SAB flips the slot + to 'Completed' with a real ``storage`` path — no matter how long PP + takes — then returns it.""" + from core.download_plugins.album_bundle import poll_album_download + + adapter = _sab_with_config() + downloading = _mock_response(200, {'queue': {'slots': [ + {'nzo_id': 'hd', 'filename': 'Hunky Dory', 'status': 'Downloading', + 'percentage': '99', 'mb': '1700', 'mbleft': '17', 'timeleft': '0:00:05'}, + ]}}) + empty_queue = _mock_response(200, {'queue': {'slots': []}}) + # Long post-processing window, reported via HISTORY (download already + # left the queue). storage NOT yet written. + pp_verify = _mock_response(200, {'history': {'slots': [ + {'nzo_id': 'hd', 'name': 'Hunky Dory', 'status': 'Verifying', 'bytes': 1_700_000_000}, + ]}}) + pp_extract = _mock_response(200, {'history': {'slots': [ + {'nzo_id': 'hd', 'name': 'Hunky Dory', 'status': 'Extracting', 'bytes': 1_700_000_000}, + ]}}) + done = _mock_response(200, {'history': {'slots': [ + {'nzo_id': 'hd', 'name': 'Hunky Dory', 'status': 'Completed', + 'bytes': 1_700_000_000, + 'storage': '/data/downloads/music/David.Bowie-Hunky.Dory'}, + ]}}) + + # Each poll = queue call + (if empty) history call. + poll_results = [ + downloading, # poll 1: queue has it @99% (1 call) + empty_queue, pp_verify, # poll 2: PP verifying in history + empty_queue, pp_verify, # poll 3: still verifying + empty_queue, pp_extract, # poll 4: extracting + empty_queue, pp_extract, # poll 5: still extracting + empty_queue, pp_extract, # poll 6: still extracting (past old 5-poll budget) + empty_queue, pp_extract, # poll 7 + empty_queue, done, # poll 8: completed + storage + ] + + class _Clock: + def __init__(self): self.now = 0.0 + def monotonic(self): return self.now + def sleep(self, s): self.now += s + + clock = _Clock() + emits: list = [] + with patch('core.usenet_clients.sabnzbd.http_requests.get', + side_effect=poll_results): + result = poll_album_download( + get_status=lambda: adapter._get_status_sync('hd'), + title='David Bowie - Hunky Dory', + emit=lambda state, **kw: emits.append((state, kw)), + complete_states=frozenset(['completed']), + failed_states=frozenset(['failed']), + transient_miss_threshold=5, + # Short no-path budget proves PP is NOT consuming it anymore. + completed_no_path_threshold=2, + sleep=clock.sleep, monotonic=clock.monotonic, + poll_interval=2.0, timeout=600.0, + ) + + assert result == '/data/downloads/music/David.Bowie-Hunky.Dory' + # Never failed despite PP outlasting both the miss budget AND the + # (deliberately tiny) completed-no-path budget — PP is non-terminal. + assert 'failed' not in [e[0] for e in emits] + + # --------------------------------------------------------------------------- # NZBGet # --------------------------------------------------------------------------- From d2a730a6aa0d73f4389d0949b6aeae52d733fe04 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 22:38:55 -0700 Subject: [PATCH 21/52] UI consistency (page shell 1/2): extract shared .page-shell primitive First step of the page-layout-shell standardization (kettui's UI-consistency point #1). The dashboard, tools, watchlist and wishlist pages each defined a byte-identical "card" container (padding 28px 24px 30px, margin 20px, gradient bg, radius 24px, border + border-top, layered shadow) under four different class names. Extract that into a single `.page-shell` primitive (modeled on the canonical dashboard/stats look) and have the four pages adopt it. Each keeps its bespoke class for page-specific extras and as a JS/mobile hook: - dashboard-container: keeps display:flex / column / gap:25px - watchlist/wishlist-page-container: keep position:relative - tools-page-container: no extras (box now fully from .page-shell) Zero visual change: computed styles are identical (declarations relocated, not altered), and mobile.css overrides still target the retained bespoke classes. Per-page themed headers (watchlist amber, etc.) are intentionally NOT touched. The class name is intended for reuse by the React pages too, so the primitive is shared across both stacks. Next (wave 2): migrate settings / automations / playlist-explorer / library onto .page-shell, which snaps their slightly-off spacing to canonical. --- webui/index.html | 8 +++--- webui/static/style.css | 63 ++++++++++++------------------------------ 2 files changed, 21 insertions(+), 50 deletions(-) diff --git a/webui/index.html b/webui/index.html index c01545e8..d067db76 100644 --- a/webui/index.html +++ b/webui/index.html @@ -311,7 +311,7 @@
-
+

System Dashboard

@@ -6322,7 +6322,7 @@ TOOLS PAGE ═══════════════════════════════════════════════════════════════════ -->
-
+

@@ -6779,7 +6779,7 @@ WATCHLIST PAGE ═══════════════════════════════════════════════════════════════════ -->
-
+
@@ -6895,7 +6895,7 @@ WISHLIST PAGE ═══════════════════════════════════════════════════════════════════ -->
-
+
diff --git a/webui/static/style.css b/webui/static/style.css index 7c2aa9d2..a5fedaaf 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -7990,13 +7990,13 @@ body.helper-mode-active #dashboard-activity-feed:hover { /* ======================================================= */ /* Main Dashboard Layout */ -.dashboard-container { - display: flex; - flex-direction: column; - gap: 25px; - /* Spacing between sections */ - padding: 28px 24px 30px 24px; - +/* ── Shared page shell ────────────────────────────────────────────── + The canonical page card (the dashboard / stats look). Adopted by the + dashboard, tools, watchlist and wishlist pages — and intended for reuse + by the React pages too (same class name). Page-specific extras (flex + layout, position) stay on the per-page class. */ +.page-shell { + padding: 28px 24px 30px; /* Semi-transparent to let page particles show through */ background: linear-gradient(135deg, rgba(20, 20, 20, 0.55) 0%, @@ -8005,7 +8005,6 @@ body.helper-mode-active #dashboard-activity-feed:hover { border: 1px solid rgba(255, 255, 255, 0.08); border-top: 1px solid rgba(255, 255, 255, 0.12); margin: 20px; - /* Soft floating shadow */ box-shadow: 0 8px 32px rgba(0, 0, 0, 0.3), @@ -8013,6 +8012,13 @@ body.helper-mode-active #dashboard-activity-feed:hover { inset 0 1px 0 rgba(255, 255, 255, 0.08); } +.dashboard-container { + display: flex; + flex-direction: column; + gap: 25px; + /* Spacing between sections; card styling comes from .page-shell */ +} + .dashboard-section { display: flex; flex-direction: column; @@ -59446,20 +59452,7 @@ body.reduce-effects *::after { TOOLS PAGE ═══════════════════════════════════════════════════════════════════ */ -.tools-page-container { - padding: 28px 24px 30px; - margin: 20px; - background: linear-gradient(135deg, - rgba(20, 20, 20, 0.55) 0%, - rgba(12, 12, 12, 0.62) 100%); - border-radius: 24px; - border: 1px solid rgba(255, 255, 255, 0.08); - border-top: 1px solid rgba(255, 255, 255, 0.12); - box-shadow: - 0 8px 32px rgba(0, 0, 0, 0.3), - 0 4px 16px rgba(0, 0, 0, 0.2), - inset 0 1px 0 rgba(255, 255, 255, 0.08); -} +/* .tools-page-container: card styling now from .page-shell (no extras). */ .tools-page-header { padding: 20px 24px 18px; @@ -59632,18 +59625,7 @@ body.reduce-effects *::after { ═══════════════════════════════════════════════════════════════════ */ .watchlist-page-container { - padding: 28px 24px 30px; - margin: 20px; - background: linear-gradient(135deg, - rgba(20, 20, 20, 0.55) 0%, - rgba(12, 12, 12, 0.62) 100%); - border-radius: 24px; - border: 1px solid rgba(255, 255, 255, 0.08); - border-top: 1px solid rgba(255, 255, 255, 0.12); - box-shadow: - 0 8px 32px rgba(0, 0, 0, 0.3), - 0 4px 16px rgba(0, 0, 0, 0.2), - inset 0 1px 0 rgba(255, 255, 255, 0.08); + /* card styling from .page-shell; keep page-specific positioning */ position: relative; } @@ -60044,18 +60026,7 @@ body.reduce-effects *::after { ═══════════════════════════════════════════════════════════════════ */ .wishlist-page-container { - padding: 28px 24px 30px; - margin: 20px; - background: linear-gradient(135deg, - rgba(20, 20, 20, 0.55) 0%, - rgba(12, 12, 12, 0.62) 100%); - border-radius: 24px; - border: 1px solid rgba(255, 255, 255, 0.08); - border-top: 1px solid rgba(255, 255, 255, 0.12); - box-shadow: - 0 8px 32px rgba(0, 0, 0, 0.3), - 0 4px 16px rgba(0, 0, 0, 0.2), - inset 0 1px 0 rgba(255, 255, 255, 0.08); + /* card styling from .page-shell; keep page-specific positioning */ position: relative; } From def58a99076d354b6bcf6d5b535fde82c373e1a1 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 22:54:31 -0700 Subject: [PATCH 22/52] UI consistency (page shell 2/N): automations adopts .page-shell card First of the "flat -> card" conversions. The automations list view sat directly on the page background (.automations-container = bare padding) while its inner .dashboard-header is the same header dashboard uses. Adopt .page-shell so the page becomes a floating gradient card structurally identical to the dashboard (page-shell card > dashboard-header > content). - Drop .automations-container's bespoke `padding: 20px 24px` (card padding now from .page-shell); keep the class as the mobile/JS hook. - Add `page-shell` to the container in markup. Visible change by design (this page was not previously a card). Mobile keeps its existing .automations-container padding override. --- webui/index.html | 2 +- webui/static/style.css | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/webui/index.html b/webui/index.html index d067db76..1f60ab60 100644 --- a/webui/index.html +++ b/webui/index.html @@ -2280,7 +2280,7 @@
-
+

Automations

diff --git a/webui/static/style.css b/webui/static/style.css index a5fedaaf..3846e676 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -43838,7 +43838,7 @@ body.downloads-disabled [onclick*="DownloadMissing"]:not([onclick*="close"]) { AUTOMATIONS PAGE =========================== */ -.automations-container { padding: 20px 24px; } +/* .automations-container: card styling from .page-shell */ .automations-list { display: flex; flex-direction: column; gap: 0; } #automations-list-view { padding-bottom: 10vh; } From 45bbc99d94b0728ddda9dd5c047d18c547c24c24 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 22:57:09 -0700 Subject: [PATCH 23/52] UI consistency (page shell 3/N): playlist-explorer adopts .page-shell card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert the playlist-explorer page from a flat padded container to the .page-shell floating card. Drop its bespoke `padding: 24px 32px`; keep the full-height flex layout (display:flex / column / min-height:100%) since the explorer fills the viewport. Visible change by design. Watch: the full-height min-height:100% inside a margin:20px card may run slightly tall — to be confirmed visually. --- webui/index.html | 2 +- webui/static/style.css | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/webui/index.html b/webui/index.html index 1f60ab60..d307039c 100644 --- a/webui/index.html +++ b/webui/index.html @@ -3632,7 +3632,7 @@
-
+
diff --git a/webui/static/style.css b/webui/static/style.css index 3846e676..276ca321 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -54451,7 +54451,7 @@ tr.tag-diff-same { ================================================================================== */ .explorer-container { - padding: 24px 32px; + /* card styling from .page-shell; keep full-height flex layout */ display: flex; flex-direction: column; min-height: 100%; From 0b325da3e980d1593c02b0ad68a4aac2cbf60eb3 Mon Sep 17 00:00:00 2001 From: Tyler Richardson-LaPlume <170156756+IamGroot60@users.noreply.github.com> Date: Fri, 29 May 2026 02:04:47 -0400 Subject: [PATCH 24/52] =?UTF-8?q?Usenet=20bundle:=20writable=20staging=20d?= =?UTF-8?q?ir=20+=20client=E2=86=92local=20path=20resolution=20(#721)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the poll fix, covering the two things that blocked a successful end-to-end album import once the poll itself stopped freezing: 1. Staging dir permissions The album-bundle private staging path defaults to 'storage/album_bundle_staging' -> /app/storage, but /app/storage was never created or chowned by the image (unlike /app/Staging, /app/Transfer, etc.), and /app is root-owned. The copy failed with "[Errno 13] Permission denied: 'storage'" under the non-root soulsync UID. Added /app/storage to the Dockerfile build-time mkdir+chown and the entrypoint PUID/PGID chown, exactly like the sibling runtime dirs. 2. Client->local path resolution Usenet/torrent clients report save paths from inside THEIR OWN container (e.g. SAB '/data/downloads/music/'); SoulSync often mounts the same files at a different point ('/app/downloads/'). Feeding the client path straight to the audio walker yields "No audio files found" though the files are physically present. New resolve_reported_save_path(): a. use the reported path as-is if readable (mirrored mounts), b. apply explicit download_source.usenet_path_mappings ({from,to}, Sonarr/Radarr-style) for non-shared layouts, c. basename fallback under SoulSync's own download roots — zero-config for the standard shared-volume arr setup. Wired into both call sites in usenet.py AND torrent.py (download_album_to_staging + _finalize_download), logging any translation and including the resolved path in the no-audio error. Tests: resolver verbatim / explicit-mapping / basename-fallback / priority / not-found / empty / mapping-miss-then-basename. ruff + compileall + pytest green (645 in the download suites). Co-Authored-By: Claude Opus 4.8 (1M context) --- Dockerfile | 11 ++- core/download_plugins/album_bundle.py | 106 ++++++++++++++++++++++++++ core/download_plugins/torrent.py | 22 +++++- core/download_plugins/usenet.py | 23 +++++- entrypoint.sh | 2 +- tests/test_album_bundle.py | 87 +++++++++++++++++++++ 6 files changed, 240 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index fdf5a21f..375b4fd9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,8 +79,15 @@ COPY --chown=soulsync:soulsync --from=webui-builder /app/webui/static/dist /app/ # fails silently on rootless Docker where the soulsync UID can't write # to /app — playback then errors out with no obvious cause. Pre-baking # at build time (when the layer is owned by root) avoids that path. -RUN mkdir -p /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream /app/MusicVideos /app/scripts && \ - chown soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream /app/MusicVideos /app/scripts +# NOTE: /app/storage is the PRIVATE album-bundle staging area for the +# torrent / usenet whole-release flow (download_source.album_bundle_staging_path +# defaults to 'storage/album_bundle_staging'). Like /app/Stream it's created +# lazily at runtime via mkdir(parents=True); without pre-baking it owned by +# soulsync, the album-bundle copy fails with "[Errno 13] Permission denied: +# 'storage'" because /app itself is root-owned and the soulsync UID can't +# create a top-level dir there. +RUN mkdir -p /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream /app/storage /app/MusicVideos /app/scripts && \ + chown soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream /app/storage /app/MusicVideos /app/scripts # Create defaults directory and copy template files # These will be used by entrypoint.sh to initialize empty volumes diff --git a/core/download_plugins/album_bundle.py b/core/download_plugins/album_bundle.py index 6abe6328..cf9030e4 100644 --- a/core/download_plugins/album_bundle.py +++ b/core/download_plugins/album_bundle.py @@ -487,6 +487,111 @@ def poll_album_download( return None +def _candidate_download_roots(config_get: Callable[..., Any]) -> list: + """Directories where THIS process can read finished downloads — used by + ``resolve_reported_save_path`` for the basename fallback. + + Order matters: most-specific usenet/torrent roots first, then the + general Soulseek download / transfer dirs, which in the standard + shared-volume arr setup are bind-mounted to the very directory the + usenet client writes its completed downloads into. Relative values + (e.g. ``./downloads``) resolve against the process CWD — the + container's ``/app`` — which is exactly where those mounts live. + """ + roots: list = [] + for key in ( + 'download_source.usenet_download_path', + 'usenet_client.completed_path', + 'usenet_client.download_path', + 'download_source.torrent_download_path', + 'soulseek.download_path', + 'soulseek.transfer_path', + ): + value = config_get(key, None) + if value: + roots.append(str(value)) + seen: set = set() + out: list = [] + for root in roots: + if root not in seen: + seen.add(root) + out.append(root) + return out + + +def resolve_reported_save_path( + reported_path: Optional[str], + config_get: Optional[Callable[..., Any]] = None, +) -> Optional[str]: + """Translate a downloader-reported save_path into one THIS process can read. + + Usenet / torrent clients report paths from inside THEIR OWN container + (e.g. SAB hands back ``/data/downloads/music/``); SoulSync often + mounts the very same files at a different point (``/app/downloads/``). + Feeding the client's path straight to the audio walker then yields + "No audio files found" even though the files are physically present — + the classic arr-stack remote-path mismatch. + + Resolution order: + 1. The reported path verbatim, if it's a readable directory here + (deployments that mirror the client's mount paths). + 2. Explicit prefix mappings from ``download_source.usenet_path_mappings`` + — a list of ``{"from": "...", "to": "..."}`` (Sonarr/Radarr-style + remote path mapping) for non-shared / oddly-mounted layouts. + 3. Basename fallback: a same-named folder under a known SoulSync + download root. Zero-config for the standard shared-volume setup — + the album folder shows up under SoulSync's own ``./downloads`` + mount with the same name the client reported. + + Returns the best resolved path, or ``reported_path`` unchanged when + nothing better is found (so the caller's existing "no audio" error still + surfaces, with both paths logged). + """ + if not reported_path: + return reported_path + if config_get is None: + config_get = config_manager.get + + def _is_dir(candidate) -> bool: + try: + return Path(candidate).is_dir() + except OSError: + return False + + # 1. Reported path is directly readable — mounts already line up. + if _is_dir(reported_path): + return reported_path + + normalized = str(reported_path).replace('\\', '/') + + # 2. Explicit prefix mappings (remote-path-mapping escape hatch). + mappings = config_get('download_source.usenet_path_mappings', None) or [] + if isinstance(mappings, (list, tuple)): + for mapping in mappings: + if not isinstance(mapping, dict): + continue + frm = str(mapping.get('from') or '').replace('\\', '/').rstrip('/') + to = str(mapping.get('to') or '') + if not frm or not to: + continue + if normalized == frm or normalized.startswith(frm + '/'): + rest = normalized[len(frm):].lstrip('/') + candidate = str(Path(to) / rest) if rest else to + if _is_dir(candidate): + return candidate + + # 3. Basename fallback under known download roots — covers the standard + # shared-volume layout with zero configuration. + basename = Path(normalized).name + if basename: + for root in _candidate_download_roots(config_get): + candidate = Path(root) / basename + if _is_dir(candidate): + return str(candidate) + + return reported_path + + def copy_audio_files_atomically( sources: Iterable[Path], staging_dir: Path, ) -> list: @@ -525,6 +630,7 @@ __all__ = [ "get_poll_interval", "get_poll_timeout", "get_transient_miss_threshold", + "resolve_reported_save_path", "pick_best_album_release", "poll_album_download", "quality_score", diff --git a/core/download_plugins/torrent.py b/core/download_plugins/torrent.py index 6c7cf4c3..17ef566b 100644 --- a/core/download_plugins/torrent.py +++ b/core/download_plugins/torrent.py @@ -65,6 +65,7 @@ from core.download_plugins.album_bundle import ( get_poll_timeout, pick_best_album_release, poll_album_download, + resolve_reported_save_path, ) from core.download_plugins.base import DownloadSourcePlugin from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult @@ -354,13 +355,21 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): if not save_path: self._mark_error(download_id, "Torrent completed but no save_path reported") return + # Resolve the client-reported path to one this process can read + # (the client may report its own container's mount). See + # ``resolve_reported_save_path``. + local_path = resolve_reported_save_path(save_path) + if local_path != save_path: + logger.info("Torrent %s: resolved client path %r -> %r", + download_id[:8], save_path, local_path) try: - audio_files = collect_audio_after_extraction(Path(save_path)) + audio_files = collect_audio_after_extraction(Path(local_path)) except Exception as e: self._mark_error(download_id, f"Post-extract walk failed: {e}") return if not audio_files: - self._mark_error(download_id, f"No audio files found in {save_path}") + suffix = f" (resolved: {local_path})" if local_path != save_path else "" + self._mark_error(download_id, f"No audio files found in {save_path}{suffix}") return primary = audio_files[0] with self._lock: @@ -539,13 +548,18 @@ class TorrentDownloadPlugin(DownloadSourcePlugin): # Phase 4: extract + walk + copy to staging. _emit('staging', release=picked.title) + # Resolve the client-reported path to one this process can read. + local_path = resolve_reported_save_path(save_path) + if local_path != save_path: + logger.info("[Torrent album] Resolved client path %r -> %r", save_path, local_path) try: - audio_files = collect_audio_after_extraction(Path(save_path)) + audio_files = collect_audio_after_extraction(Path(local_path)) except Exception as e: result['error'] = f'Failed to walk audio files: {e}' return result if not audio_files: - result['error'] = f'No audio files found in {save_path}' + suffix = f' (resolved: {local_path})' if local_path != save_path else '' + result['error'] = f'No audio files found in {save_path}{suffix}' return result copied = copy_audio_files_atomically(audio_files, Path(staging_dir)) diff --git a/core/download_plugins/usenet.py b/core/download_plugins/usenet.py index 75302c50..a8b0c529 100644 --- a/core/download_plugins/usenet.py +++ b/core/download_plugins/usenet.py @@ -27,6 +27,7 @@ from core.download_plugins.album_bundle import ( get_completed_no_path_window_seconds, pick_best_album_release, poll_album_download, + resolve_reported_save_path, ) from core.download_plugins.base import DownloadSourcePlugin from core.download_plugins.torrent import ( @@ -337,13 +338,21 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): if not save_path: self._mark_error(download_id, "Usenet job completed but no save_path reported") return + # Translate the client-reported path to one THIS process can read + # (SAB reports its own container path; SoulSync may see the same + # files at a different mount). See ``resolve_reported_save_path``. + local_path = resolve_reported_save_path(save_path) + if local_path != save_path: + logger.info("Usenet %s: resolved client path %r -> %r", + download_id[:8], save_path, local_path) try: - audio_files = collect_audio_after_extraction(Path(save_path)) + audio_files = collect_audio_after_extraction(Path(local_path)) except Exception as e: self._mark_error(download_id, f"Post-extract walk failed: {e}") return if not audio_files: - self._mark_error(download_id, f"No audio files found in {save_path}") + suffix = f" (resolved: {local_path})" if local_path != save_path else "" + self._mark_error(download_id, f"No audio files found in {save_path}{suffix}") return primary = audio_files[0] with self._lock: @@ -497,13 +506,19 @@ class UsenetDownloadPlugin(DownloadSourcePlugin): return result _emit('staging', release=picked.title) + # SAB reports its own container path; SoulSync may mount the same + # files elsewhere. Resolve to a locally-readable path before walking. + local_path = resolve_reported_save_path(save_path) + if local_path != save_path: + logger.info("[Usenet album] Resolved client path %r -> %r", save_path, local_path) try: - audio_files = collect_audio_after_extraction(Path(save_path)) + audio_files = collect_audio_after_extraction(Path(local_path)) except Exception as e: result['error'] = f'Failed to walk audio files: {e}' return result if not audio_files: - result['error'] = f'No audio files found in {save_path}' + suffix = f' (resolved: {local_path})' if local_path != save_path else '' + result['error'] = f'No audio files found in {save_path}{suffix}' return result copied = copy_audio_files_atomically(audio_files, Path(staging_dir)) diff --git a/entrypoint.sh b/entrypoint.sh index 8cde8e0f..28bae10a 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -40,7 +40,7 @@ if [ "$CURRENT_UID" != "$PUID" ] || [ "$CURRENT_GID" != "$PGID" ]; then DATA_OWNER=$(stat -c '%u:%g' /app/data 2>/dev/null || echo "unknown") if [ "$DATA_OWNER" != "$PUID:$PGID" ]; then echo "🔒 Fixing permissions on app directories..." - chown -R soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream 2>/dev/null || true + chown -R soulsync:soulsync /app/config /app/data /app/logs /app/downloads /app/Transfer /app/Staging /app/Stream /app/storage 2>/dev/null || true else echo "✅ App directory permissions already correct" fi diff --git a/tests/test_album_bundle.py b/tests/test_album_bundle.py index 57790286..6a237b22 100644 --- a/tests/test_album_bundle.py +++ b/tests/test_album_bundle.py @@ -32,6 +32,7 @@ from core.download_plugins.album_bundle import ( get_poll_timeout, pick_best_album_release, quality_score, + resolve_reported_save_path, unique_staging_path, ) @@ -325,6 +326,92 @@ def test_get_completed_no_path_window_falls_back_on_garbage() -> None: assert get_completed_no_path_window_seconds() == DEFAULT_COMPLETED_NO_PATH_WINDOW_SECONDS +# --------------------------------------------------------------------------- +# resolve_reported_save_path — downloader→local path translation. The arr +# remote-path problem: SAB reports its own container path, SoulSync mounts +# the same files elsewhere. +# --------------------------------------------------------------------------- + + +def _cfg(values: dict): + """Build a config_manager.get-shaped callable from a dict.""" + def _get(key, default=None): + return values.get(key, default) + return _get + + +def test_resolve_returns_reported_path_verbatim_when_readable(tmp_path: Path) -> None: + """If the client's path is already readable here (mounts mirror the + client), return it unchanged — no translation needed.""" + album = tmp_path / "MyAlbum" + album.mkdir() + # config_get should never even be consulted on the happy path. + assert resolve_reported_save_path(str(album), config_get=_cfg({})) == str(album) + + +def test_resolve_uses_explicit_prefix_mapping(tmp_path: Path) -> None: + """Sonarr/Radarr-style remote path mapping: SAB's prefix is rewritten + to a SoulSync-visible root.""" + (tmp_path / "MyAlbum").mkdir() + cfg = _cfg({'download_source.usenet_path_mappings': [ + {'from': '/data/downloads/music', 'to': str(tmp_path)}, + ]}) + resolved = resolve_reported_save_path('/data/downloads/music/MyAlbum', config_get=cfg) + assert resolved == str(tmp_path / "MyAlbum") + + +def test_resolve_basename_fallback_against_download_root(tmp_path: Path) -> None: + """Zero-config shared-volume case: the album folder shows up under + SoulSync's own download root with the same name SAB reported.""" + (tmp_path / "MyAlbum").mkdir() + cfg = _cfg({'soulseek.download_path': str(tmp_path)}) + resolved = resolve_reported_save_path('/data/downloads/music/MyAlbum', config_get=cfg) + assert resolved == str(tmp_path / "MyAlbum") + + +def test_resolve_mapping_takes_priority_over_basename(tmp_path: Path) -> None: + """An explicit mapping that resolves wins over the basename scan.""" + mapped_root = tmp_path / "mapped" + dl_root = tmp_path / "dl" + (mapped_root / "MyAlbum").mkdir(parents=True) + (dl_root / "MyAlbum").mkdir(parents=True) + cfg = _cfg({ + 'download_source.usenet_path_mappings': [ + {'from': '/data/downloads/music', 'to': str(mapped_root)}, + ], + 'soulseek.download_path': str(dl_root), + }) + resolved = resolve_reported_save_path('/data/downloads/music/MyAlbum', config_get=cfg) + assert resolved == str(mapped_root / "MyAlbum") + + +def test_resolve_returns_reported_unchanged_when_nothing_found(tmp_path: Path) -> None: + """No readable path, no mapping hit, no basename match → return the + original so the caller's 'no audio' error still surfaces.""" + cfg = _cfg({'soulseek.download_path': str(tmp_path)}) # empty root + reported = '/data/downloads/music/Missing' + assert resolve_reported_save_path(reported, config_get=cfg) == reported + + +def test_resolve_handles_empty_and_none(tmp_path: Path) -> None: + assert resolve_reported_save_path('', config_get=_cfg({})) == '' + assert resolve_reported_save_path(None, config_get=_cfg({})) is None + + +def test_resolve_skips_mapping_when_target_missing_then_tries_basename(tmp_path: Path) -> None: + """A mapping whose translated path doesn't exist must not short-circuit + — fall through to the basename scan.""" + (tmp_path / "MyAlbum").mkdir() + cfg = _cfg({ + 'download_source.usenet_path_mappings': [ + {'from': '/data/downloads/music', 'to': '/nope/not/mounted'}, + ], + 'soulseek.download_path': str(tmp_path), + }) + resolved = resolve_reported_save_path('/data/downloads/music/MyAlbum', config_get=cfg) + assert resolved == str(tmp_path / "MyAlbum") + + # --------------------------------------------------------------------------- # poll_album_download — lifted poll loop for both torrent + usenet plugins. # --------------------------------------------------------------------------- From 44faf44fca4fba6161154ac3691e53ab8a0f0e64 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 23:23:16 -0700 Subject: [PATCH 25/52] UI consistency (page shell 5/N): settings adopts .page-shell card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings was the one flat page with no single wrapper — its .dashboard-header and .settings-content sat as siblings directly under .page. Wrap both in a single .page-shell div so the page becomes a floating card with the header banner at the top, matching the dashboard structure. HTML-only change (no CSS: .settings-content keeps its minor `0 4px` inner padding). Library is intentionally NOT converted — its full-height artist grid + A-Z jump rail overflow a margin:20px card, so it stays flat as a documented exception (same category as search/discover/active-downloads). --- webui/index.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webui/index.html b/webui/index.html index d307039c..c5d5f2bd 100644 --- a/webui/index.html +++ b/webui/index.html @@ -3735,6 +3735,7 @@
+

Settings

@@ -6200,6 +6201,7 @@
+
From eebc58d3ff0e5ae2dbe7a49de274edeaf665cbd9 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 23:40:10 -0700 Subject: [PATCH 26/52] UI consistency (buttons 1/N): add shared .btn primitive; migrate config-modal Start of the button-consolidation pass (kettui's #1). The app had ~236 button classes / ~8-10 distinct looks with heavy near-duplication. Introduce a canonical .btn design-system primitive (base + .btn--primary / .btn--secondary / .btn--danger), modeled on the dominant existing look (accent-gradient primary, translucent ghost, semantic danger) and built on the accent CSS vars. New markup and the React pages should use this; existing per-page button classes will migrate onto it family by family. First family migrated: the config/settings modal buttons (.config-modal-btn*, 4 static uses, no JS refs) -> .btn .btn--primary / .btn--secondary. Removed the now-dead .config-modal-btn* rules and re-scoped its mobile full-width override to `.config-modal-actions .btn`. Visible change is minor by design (padding 28->24px, gradient direction normalized). Proof step for sign-off on the .btn look before rolling wider. --- webui/index.html | 8 ++-- webui/static/style.css | 103 ++++++++++++++++++++++++----------------- 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/webui/index.html b/webui/index.html index c5d5f2bd..55be3e95 100644 --- a/webui/index.html +++ b/webui/index.html @@ -7467,11 +7467,11 @@
- -
@@ -7651,11 +7651,11 @@
- - diff --git a/webui/static/style.css b/webui/static/style.css index 276ca321..9560a68c 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -7990,6 +7990,64 @@ body.helper-mode-active #dashboard-activity-feed:hover { /* ======================================================= */ /* Main Dashboard Layout */ +/* ── Shared button primitive ──────────────────────────────────────── + The canonical button design system. New markup (and the React pages) + should use `.btn` + a modifier; existing per-page button classes are being + migrated onto it family by family. Values reflect the dominant existing + look (accent-gradient primary, translucent ghost, semantic danger/warning). + Filter "pills" / tab buttons are a separate primitive (see .tab below). */ +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 8px; + padding: 12px 24px; + border-radius: 12px; + font-size: 14px; + font-weight: 600; + font-family: inherit; + line-height: 1; + cursor: pointer; + border: none; + outline: none; + transition: all 0.2s ease; +} +.btn:disabled { opacity: 0.5; cursor: not-allowed; transform: none; } + +.btn--primary { + background: linear-gradient(135deg, rgb(var(--accent-rgb)), rgb(var(--accent-light-rgb))); + color: #000; + font-weight: 700; + box-shadow: 0 4px 16px rgba(var(--accent-rgb), 0.3); +} +.btn--primary:hover:not(:disabled) { + transform: translateY(-1px); + box-shadow: 0 6px 20px rgba(var(--accent-rgb), 0.4); +} +.btn--primary:active:not(:disabled) { transform: translateY(0); } + +.btn--secondary { + background: rgba(255, 255, 255, 0.08); + color: #fff; + border: 1px solid rgba(255, 255, 255, 0.15); +} +.btn--secondary:hover:not(:disabled) { + background: rgba(255, 255, 255, 0.12); + border-color: rgba(255, 255, 255, 0.25); +} + +.btn--danger { + background: linear-gradient(135deg, #dc2626, #ef4444); + color: #fff; + font-weight: 700; + box-shadow: 0 4px 16px rgba(220, 38, 38, 0.3); +} +.btn--danger:hover:not(:disabled) { + transform: translateY(-1px); + box-shadow: 0 6px 20px rgba(220, 38, 38, 0.4); +} + + /* ── Shared page shell ────────────────────────────────────────────── The canonical page card (the dashboard / stats look). Adopted by the dashboard, tools, watchlist and wishlist pages — and intended for reuse @@ -20880,48 +20938,7 @@ body.helper-mode-active #dashboard-activity-feed:hover { justify-content: flex-end; } -.config-modal-btn { - padding: 12px 28px; - border-radius: 12px; - font-size: 15px; - font-weight: 600; - cursor: pointer; - transition: all 0.2s ease; - border: none; - outline: none; -} - -.config-modal-btn-primary { - background: linear-gradient(135deg, rgb(var(--accent-light-rgb)), rgb(var(--accent-rgb))); - color: #000; - box-shadow: 0 4px 16px rgba(var(--accent-rgb), 0.3); -} - -.config-modal-btn-primary:hover { - background: linear-gradient(135deg, #1fdf64, rgb(var(--accent-light-rgb))); - box-shadow: 0 6px 20px rgba(var(--accent-rgb), 0.4); - transform: translateY(-1px); -} - -.config-modal-btn-primary:active { - transform: translateY(0); -} - -.config-modal-btn-primary:disabled { - opacity: 0.5; - cursor: not-allowed; -} - -.config-modal-btn-secondary { - background: rgba(255, 255, 255, 0.08); - color: #ffffff; - border: 1px solid rgba(255, 255, 255, 0.15); -} - -.config-modal-btn-secondary:hover { - background: rgba(255, 255, 255, 0.12); - border-color: rgba(255, 255, 255, 0.25); -} +/* .config-modal-btn* migrated to the shared .btn / .btn--primary / .btn--secondary primitive. */ /* Linked Provider Artist */ /* Per-source linked rows */ @@ -21159,7 +21176,7 @@ body.helper-mode-active #dashboard-activity-feed:hover { flex-direction: column-reverse; } - .config-modal-btn { + .config-modal-actions .btn { width: 100%; } } From dd5fe844d479e394ca4aa5449bad9da60887afaf Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 28 May 2026 23:48:23 -0700 Subject: [PATCH 27/52] UI consistency (buttons 2/N): wishlist modal buttons -> .btn Migrate the wishlist add-to-wishlist modal buttons onto the shared .btn primitive: primary -> .btn--primary, secondary -> .btn--secondary, the green download CTA -> new .btn--download modifier. Added a shared .btn.loading state (amber pulse, reusing the existing pulse-loading keyframe) since confirm-add-to-wishlist-btn toggles `loading` via JS (wishlist-tools.js). Removed the dead .wishlist-modal-btn* rules and re-scoped the mobile full-width override to `.wishlist-modal-actions .btn`. --- webui/index.html | 6 +-- webui/static/style.css | 96 ++++++++++-------------------------------- 2 files changed, 26 insertions(+), 76 deletions(-) diff --git a/webui/index.html b/webui/index.html index 55be3e95..bae77d96 100644 --- a/webui/index.html +++ b/webui/index.html @@ -7290,15 +7290,15 @@
- - -
diff --git a/webui/static/style.css b/webui/static/style.css index 9560a68c..8293d395 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -8047,6 +8047,26 @@ body.helper-mode-active #dashboard-activity-feed:hover { box-shadow: 0 6px 20px rgba(220, 38, 38, 0.4); } +/* Spotify-green download CTA (intentional brand color). */ +.btn--download { + background: linear-gradient(135deg, #1db954, #1ed760); + color: #fff; + border: 1px solid rgba(29, 185, 84, 0.3); +} +.btn--download:hover:not(:disabled) { + background: linear-gradient(135deg, #1ed760, #1db954); + transform: translateY(-1px); + box-shadow: 0 8px 24px rgba(29, 185, 84, 0.4); +} + +/* Shared loading state (JS toggles `.loading`). */ +.btn.loading { + background: rgba(255, 193, 7, 0.9); + color: #000; + animation: pulse-loading 1.5s infinite; + cursor: not-allowed; +} + /* ── Shared page shell ────────────────────────────────────────────── The canonical page card (the dashboard / stats look). Adopted by the @@ -28152,78 +28172,8 @@ div.artist-hero-badge { gap: 12px; } -.wishlist-modal-btn { - padding: 12px 24px; - border: none; - border-radius: 12px; - font-size: 14px; - font-weight: 600; - cursor: pointer; - transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1); - min-width: 120px; -} - -.wishlist-modal-btn-secondary { - background: rgba(255, 255, 255, 0.1); - color: rgba(255, 255, 255, 0.8); - border: 1px solid rgba(255, 255, 255, 0.2); -} - -.wishlist-modal-btn-secondary:hover { - background: rgba(255, 255, 255, 0.2); - color: #ffffff; - border-color: rgba(255, 255, 255, 0.3); -} - -.wishlist-modal-btn-download { - background: linear-gradient(135deg, #1db954, #1ed760); - color: #ffffff; - border: 1px solid rgba(29, 185, 84, 0.3); -} - -.wishlist-modal-btn-download:hover { - background: linear-gradient(135deg, #1ed760, #1db954); - transform: translateY(-1px); - box-shadow: 0 8px 24px rgba(29, 185, 84, 0.4); -} - -.wishlist-modal-btn-download:active { - transform: translateY(0); - box-shadow: 0 4px 12px rgba(29, 185, 84, 0.3); -} - -.wishlist-modal-btn-primary { - background: linear-gradient(135deg, rgb(var(--accent-rgb)) 0%, rgb(var(--accent-light-rgb)) 100%); - color: #ffffff; - border: 1px solid rgba(var(--accent-rgb), 0.3); -} - -.wishlist-modal-btn-primary:hover { - background: linear-gradient(135deg, rgb(var(--accent-light-rgb)) 0%, #22e167 100%); - transform: translateY(-1px); - box-shadow: 0 8px 24px rgba(var(--accent-rgb), 0.4); -} - -.wishlist-modal-btn-primary:active { - transform: translateY(0); - box-shadow: 0 4px 12px rgba(var(--accent-rgb), 0.3); -} - -/* Loading State */ -.wishlist-modal-btn-primary.loading { - background: rgba(255, 193, 7, 0.9); - color: #000000; - animation: pulse-loading 1.5s infinite; - cursor: not-allowed; -} - -.wishlist-modal-btn-primary:disabled { - background: rgba(255, 255, 255, 0.1); - color: rgba(255, 255, 255, 0.4); - cursor: not-allowed; - transform: none; - box-shadow: none; -} +/* .wishlist-modal-btn* migrated to the shared .btn primitive + (.btn--primary / .btn--secondary / .btn--download / .btn.loading). */ /* Animation for modal appearance */ @keyframes wishlistModalFadeIn { @@ -28310,7 +28260,7 @@ div.artist-hero-badge { gap: 12px; } - .wishlist-modal-btn { + .wishlist-modal-actions .btn { width: 100%; } } From 079c169f8d882eb709e496c91e9814f452cbdb04 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 08:28:18 -0700 Subject: [PATCH 28/52] UI consistency (tabs/cards): add .tab and .card primitives (no migration) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the canonical .tab (bordered rounded-pill filter tab) and .card ("glass" content card) primitives as the documented design-system standard for new markup and the React pages — modeled on the cleanest existing looks (watchlist filter pill; dashboard service/stat card). Deliberately NOT migrating the existing tab/card components onto them: the current implementations are visually divergent and JS-coupled (active-state toggled by class name, cards built in JS), so a blind consolidation risks subtle regressions. These primitives let new/React code be consistent now; the legacy components migrate when visually verifiable / in React. Unused classes -> zero visual change to the current UI. --- webui/static/style.css | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/webui/static/style.css b/webui/static/style.css index 8293d395..0c397652 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -8068,6 +8068,51 @@ body.helper-mode-active #dashboard-activity-feed:hover { } +/* ── Shared tab / filter-pill primitive ───────────────────────────── + Canonical pill tab (the bordered rounded-pill filter look). New markup and + the React pages should use `.tab` + `.tab.active`. Existing per-page pill + classes (watchlist-filter-btn, discog-filter, adl-pill, …) are deliberately + left in place — they're JS-coupled and visually divergent, so they should be + migrated only when visually verifiable / in React, not blind. */ +.tab { + padding: 6px 16px; + border-radius: 20px; + border: 1px solid rgba(255, 255, 255, 0.1); + background: rgba(255, 255, 255, 0.05); + color: rgba(255, 255, 255, 0.5); + font-size: 13px; + font-weight: 600; + font-family: inherit; + cursor: pointer; + transition: all 0.2s ease; +} +.tab:hover:not(.active) { color: rgba(255, 255, 255, 0.8); } +.tab.active { + color: #fff; + border-color: rgba(var(--accent-rgb), 0.5); + background: rgba(var(--accent-rgb), 0.15); +} + +/* ── Shared card primitive ────────────────────────────────────────── + Canonical "glass" content card (the dashboard service/stat card look). New + markup and the React pages should use `.card`. Existing per-feature card + classes are left in place; migrate when visually verifiable / in React. */ +.card { + padding: 20px 22px; + border-radius: 16px; + border: 1px solid rgba(255, 255, 255, 0.06); + background: rgba(18, 18, 22, 0.9); + backdrop-filter: blur(12px); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.3), inset 0 1px 0 rgba(255, 255, 255, 0.15); + transition: all 0.2s ease; +} +.card:hover { + transform: translateY(-2px); + border-color: rgba(var(--accent-rgb), 0.3); + box-shadow: 0 8px 28px rgba(0, 0, 0, 0.4), inset 0 1px 0 rgba(255, 255, 255, 0.15); +} + + /* ── Shared page shell ────────────────────────────────────────────── The canonical page card (the dashboard / stats look). Adopted by the dashboard, tools, watchlist and wishlist pages — and intended for reuse From f57fc640b271c3f30064d4871d099ae4048481a7 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 08:41:38 -0700 Subject: [PATCH 29/52] UI consistency (page shell 6/N): sync page adopts .page-shell card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standardize the sync page's outer spacing to match the other pages. Like settings, its .sync-header and .sync-content-area were siblings directly under .page (no wrapper) — wrap both in a single .page-shell div so it becomes the floating card with consistent margin/padding. HTML-only change. Watch: .sync-content-area uses height:95% (grid) — fine against an auto-height card, but to be confirmed visually (library's full-height grid was the one that didn't fit a card). --- webui/index.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webui/index.html b/webui/index.html index bae77d96..4388ffda 100644 --- a/webui/index.html +++ b/webui/index.html @@ -976,6 +976,7 @@
+
@@ -2059,6 +2060,7 @@
+
From 6129ea850859b99844076c850667ec15d1dc5e78 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 08:53:03 -0700 Subject: [PATCH 30/52] UI consistency: normalize exception-page outer gutter to .page (40px) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Low-risk tidy-up for the full-bleed "exception" pages that aren't carded. Every page already gets a 40px gutter from .page, but the exception pages were piling on inconsistent extra padding (library +20px, active-downloads +28/32px, discover/docs +0) — giving accidental 60 / 68-72 / 40 effective gutters. Drop the redundant container padding on library and active-downloads so the single .page 40px gutter is the shared, intentional outer spacing across the full-width exception pages. discover (centered max-width) and docs (sidebar layout) keep their functional layout; library's mobile padding override is unaffected. --- webui/static/style.css | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/webui/static/style.css b/webui/static/style.css index 0c397652..2603908f 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -25765,10 +25765,11 @@ body.helper-mode-active #dashboard-activity-feed:hover { =============================== */ .library-container { + /* outer gutter comes from .page (40px) — shared across the full-width + exception pages so spacing is intentional, not accidental */ height: 100%; display: flex; flex-direction: column; - padding: 20px; gap: 20px; } @@ -58153,9 +58154,10 @@ body.reduce-effects *::after { } .adl-layout { + /* outer gutter comes from .page (40px) — consistent with the other + full-width exception pages (library, etc.) */ display: flex; gap: 0; - padding: 28px 32px; max-width: 1440px; } From 779d729a08bbab3a9ccf45d91d12b8caaf94dc59 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 09:16:47 -0700 Subject: [PATCH 31/52] Fix: Spotify playlist sync shows only 100 tracks (#736) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported by @Yug1900: a Spotify playlist's overview shows the correct count (e.g. 203) but the sync modal lists only 100, and only 100 would sync. Root cause in /api/spotify/playlist/ (the Spotify-tab track fetch): owned playlists fetch + paginate fine (sp.next loops over all pages). But when the authenticated playlist_items call FAILS (intermittently, often a 403 on followed / not-owned playlists) it fell straight back to the public embed scraper, which is hard-capped at ~100 tracks and has no pagination — and the result was returned as if complete. The overview total is correct because it comes from the playlist *metadata* listing, which succeeds independently. Fixes (additive — the owned/working path is unchanged): - Retry the items fetch once before resorting to the embed scraper, so a transient failure no longer silently truncates a large playlist. - Guard against silent truncation: compare fetched count against the playlist's known total; if short (or via the capped embed fallback), log a warning and return `incomplete: true` + `expected_total` instead of presenting a partial list as complete. No behavior change when the fetch succeeds in full (incomplete=false, extra fields ignored by the frontend). Lets a future UI tweak surface "got 100 of 203 — retry" rather than silently dropping tracks. --- web_server.py | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/web_server.py b/web_server.py index 17c4322e..27bca092 100644 --- a/web_server.py +++ b/web_server.py @@ -18956,11 +18956,24 @@ def get_playlist_tracks(playlist_id): # Fetch all tracks with full album data tracks = [] - try: - results = spotify_client._get_playlist_items_page(playlist_id, limit=100) - except Exception as items_err: - # 403 on followed playlists — try the public embed scraper as fallback - logger.warning(f"Playlist items fetch failed ({items_err}), trying public embed scraper") + # The items endpoint intermittently fails for followed playlists — retry + # once before resorting to the (≈100-capped) public embed scraper, so a + # transient failure doesn't silently truncate a large playlist. + results = None + items_err = None + for _attempt in range(2): + try: + results = spotify_client._get_playlist_items_page(playlist_id, limit=100) + break + except Exception as _e: + items_err = _e + if _attempt == 0: + logger.warning(f"Playlist items fetch failed ({_e}); retrying once") + time.sleep(0.5) + if results is None: + # Both attempts failed (often a 403 on followed playlists) — fall back + # to the public embed scraper as a last resort (capped at ~100 tracks). + logger.warning(f"Playlist items unavailable after retry ({items_err}), trying public embed scraper") try: from core.spotify_public_scraper import scrape_spotify_embed embed_data = scrape_spotify_embed('playlist', playlist_id) @@ -18986,7 +18999,10 @@ def get_playlist_tracks(playlist_id): 'track_count': len(tracks), 'image_url': playlist_data['images'][0]['url'] if playlist_data.get('images') else None, 'snapshot_id': playlist_data.get('snapshot_id', ''), - 'tracks': tracks + 'tracks': tracks, + # Embed scrape is capped at ~100 — mark as incomplete. + 'incomplete': True, + 'expected_total': (playlist_data.get('tracks') or {}).get('total'), } return jsonify(playlist_dict) except Exception as scrape_err: @@ -19014,6 +19030,19 @@ def get_playlist_tracks(playlist_id): else: results = None + # Guard against silent truncation: Spotify occasionally returns a short + # first page (next=None) for some playlists, so the loop above can end + # early. Compare against the playlist's known total and flag it rather + # than presenting a partial list as complete (bug #736). + expected_total = (playlist_data.get('tracks') or {}).get('total') + incomplete = bool(expected_total) and len(tracks) < expected_total + if incomplete: + logger.warning( + f"Playlist {playlist_id} ('{playlist_data.get('name')}'): fetched " + f"{len(tracks)}/{expected_total} tracks — Spotify returned incomplete " + f"items data (flagged, not silently truncated)" + ) + # Convert playlist to dict playlist_dict = { 'id': playlist_data['id'], @@ -19025,7 +19054,9 @@ def get_playlist_tracks(playlist_id): 'track_count': len(tracks), 'image_url': playlist_data['images'][0]['url'] if playlist_data.get('images') else None, 'snapshot_id': playlist_data.get('snapshot_id', ''), - 'tracks': tracks + 'tracks': tracks, + 'incomplete': incomplete, + 'expected_total': expected_total, } return jsonify(playlist_dict) except Exception as e: From 560156abeef3d2be0145b2d3ae6ee022cc434a59 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 09:30:38 -0700 Subject: [PATCH 32/52] Fix: import overwrites album-artist tag to "Unknown Artist" (#735) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported by CubeComming: importing media keeps the track artist correct (e.g. Billie Eilish) but changes the album-artist tag ("Albuminterpret") to "Unknown Artist", breaking grouping in the media server. Cause: in extract_source_metadata (core/metadata/source.py), album_artist is seeded from the resolved track artist, then overridden by the album CONTEXT's first artist. When the album lookup comes back unresolved, that first artist is the literal "Unknown Artist" placeholder — which is truthy, so it clobbered the real artist. Fix: treat "Unknown Artist" (and empty) as a non-value — only let the album context override the album_artist when it names a real artist. A genuine album artist (e.g. "Various Artists") still overrides as before. Tests: tests/metadata/test_album_artist_unknown.py — placeholder doesn't clobber, real album artist still used, no-album-context falls back to track artist, empty doesn't clobber. (Pre-existing test_album_mbid_cache.py failures are an unrelated sandbox DB disk-I/O issue.) --- core/metadata/source.py | 18 +++-- tests/metadata/test_album_artist_unknown.py | 73 +++++++++++++++++++++ 2 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 tests/metadata/test_album_artist_unknown.py diff --git a/core/metadata/source.py b/core/metadata/source.py index 65ac780c..fcd69569 100644 --- a/core/metadata/source.py +++ b/core/metadata/source.py @@ -1032,11 +1032,19 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di album_artists = album_ctx.get("artists", []) if album_artists: first_album_artist = album_artists[0] - if isinstance(first_album_artist, dict) and first_album_artist.get("name"): - raw_album_artist = first_album_artist["name"] - elif isinstance(first_album_artist, str) and first_album_artist: - raw_album_artist = first_album_artist - album_artists_for_collab = album_artists + if isinstance(first_album_artist, dict): + candidate = first_album_artist.get("name", "") + elif isinstance(first_album_artist, str): + candidate = first_album_artist + else: + candidate = "" + # An unresolved "Unknown Artist" placeholder from the album context + # must NOT clobber the real track artist already in raw_album_artist + # (bug #735: album-artist tag overwritten to "Unknown Artist" on + # import). Only override when the album context names a real artist. + if candidate and candidate != "Unknown Artist": + raw_album_artist = candidate + album_artists_for_collab = album_artists collab_mode = cfg.get("file_organization.collab_artist_mode", "first") if collab_mode == "first" and raw_album_artist: diff --git a/tests/metadata/test_album_artist_unknown.py b/tests/metadata/test_album_artist_unknown.py new file mode 100644 index 00000000..f8b96d93 --- /dev/null +++ b/tests/metadata/test_album_artist_unknown.py @@ -0,0 +1,73 @@ +"""Regression for #735 (CubeComming): importing media overwrites the +album-artist tag ('Albuminterpret') to 'Unknown Artist'. + +When the metadata source resolves the track artist correctly (e.g. Billie +Eilish) but the album CONTEXT comes back with an unresolved 'Unknown Artist' +placeholder, extract_source_metadata used to take album_ctx['artists'][0]['name'] +unconditionally — clobbering the real album artist. The fix: an 'Unknown Artist' +placeholder must not override a real artist. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + + +def _cfg(): + cfg = MagicMock() + cfg.get.side_effect = lambda key, default=None: { + "metadata_enhancement.enabled": True, + "metadata_enhancement.tags.write_multi_artist": False, + "metadata_enhancement.tags.feat_in_title": False, + "metadata_enhancement.tags.artist_separator": ", ", + "file_organization.collab_artist_mode": "first", + }.get(key, default) + return cfg + + +def _extract(context, artist_dict, album_info=None): + from core.metadata import source as src + with patch.object(src, "get_config_manager", return_value=_cfg()): + return src.extract_source_metadata(context, artist_dict, album_info or {}) + + +def test_unknown_album_artist_placeholder_does_not_clobber_real_artist(): + # Track resolves to a real artist; album context is an unresolved placeholder. + context = { + "original_search_result": {"title": "Therefore I Am", "artists": [{"name": "Billie Eilish"}]}, + "album": {"artists": [{"name": "Unknown Artist"}]}, + "source": "spotify", + } + md = _extract(context, {"name": "Billie Eilish"}) + assert md["artist"] == "Billie Eilish" + assert md["album_artist"] == "Billie Eilish" # NOT "Unknown Artist" + + +def test_real_album_artist_still_overrides(): + # A genuine album artist (not the placeholder) should still be used. + context = { + "original_search_result": {"title": "Song", "artists": [{"name": "Some Singer"}]}, + "album": {"artists": [{"name": "Various Artists"}]}, + "source": "spotify", + } + md = _extract(context, {"name": "Some Singer"}) + assert md["album_artist"] == "Various Artists" + + +def test_no_album_context_falls_back_to_track_artist(): + context = { + "original_search_result": {"title": "Song", "artists": [{"name": "Solo Act"}]}, + "source": "spotify", + } + md = _extract(context, {"name": "Solo Act"}) + assert md["album_artist"] == "Solo Act" + + +def test_empty_album_artist_does_not_clobber(): + context = { + "original_search_result": {"title": "Song", "artists": [{"name": "Real Name"}]}, + "album": {"artists": [{"name": ""}]}, + "source": "spotify", + } + md = _extract(context, {"name": "Real Name"}) + assert md["album_artist"] == "Real Name" From d9a24d48c6f812da8d485ea8fe9c76bc2f186e94 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 10:18:38 -0700 Subject: [PATCH 33/52] Fix: search results disappear when interacting with the media player (#732) Search results live in an overlay dismissed by an outside-click handler whose allow-list omitted the floating media player. Clicking the mini player to open the now-playing modal (or clicking inside that modal) registered as an outside click and tore the results down, forcing a re-search. Add the media player containers (#media-player mini bar and #np-modal-overlay expanded modal) to the dismiss allow-lists in both the Search page (search.js) and the global search widget (downloads.js), which share the same outside-click pattern. Additive change: only adds exceptions, so every existing dismiss case is unchanged. --- webui/static/downloads.js | 5 +++++ webui/static/search.js | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/webui/static/downloads.js b/webui/static/downloads.js index 714ae934..39f0cf5a 100644 --- a/webui/static/downloads.js +++ b/webui/static/downloads.js @@ -5709,6 +5709,11 @@ let _gsController = null; const freshResults = document.getElementById('gsearch-results'); const target = e.target; if (freshBar?.contains(target) || freshResults?.contains(target)) return; + // The media player (mini bar + expanded now-playing modal) + // floats above the page. Clicking it — e.g. opening the full + // modal from the mini player, or anything inside that modal — + // must NOT tear down the global search results (#732). + if (target.closest && target.closest('#media-player, #np-modal-overlay')) return; _gsDeactivate(); }, 100); }); diff --git a/webui/static/search.js b/webui/static/search.js index cd27d34e..f963f465 100644 --- a/webui/static/search.js +++ b/webui/static/search.js @@ -288,7 +288,12 @@ function initializeSearchModeToggle() { const isClickOnSourceRow = e.target.closest('#enh-source-row'); // Modal sits above the dropdown; closing it shouldn't dismiss results. const isClickInModal = e.target.closest('.download-missing-modal'); - if (!isClickInside && !isClickOnSourceRow && !isClickInModal) { + // The media player (mini bar + expanded now-playing modal) floats + // above the page. Interacting with it — e.g. clicking the mini + // player to open the full modal, or anywhere inside that modal — + // must NOT dismiss the search results (#732). + const isClickInPlayer = e.target.closest('#media-player, #np-modal-overlay'); + if (!isClickInside && !isClickOnSourceRow && !isClickInModal && !isClickInPlayer) { hideDropdown(); } } From 21426af7fe6aa5e2bd743c2458474acf637e7653 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 10:47:02 -0700 Subject: [PATCH 34/52] Tools: add Deep Scan option to the Database Updater MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Tools-page Database Updater dropdown only offered Incremental and Full Refresh, even though the backend (/api/database/update with deep_scan) and the dashboard Deep Scan button already supported a deep scan. Wire the missing option into the Tools UI: - Add a "Deep Scan" option to the #db-refresh-type dropdown. - handleDbUpdateButtonClick now sends { deep_scan: true } for that option (deep scan takes precedence server-side) and confirms first, since deep scan removes stale entries — mirroring the dashboard flow. Frontend-only; the progress/status handler already drives the bar from the backend phase ("Deep scan: ...") and the help/docs copy already described all three modes. --- webui/index.html | 1 + webui/static/sync-services.js | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/webui/index.html b/webui/index.html index 4388ffda..b37d6349 100644 --- a/webui/index.html +++ b/webui/index.html @@ -6447,6 +6447,7 @@
diff --git a/webui/static/sync-services.js b/webui/static/sync-services.js index 63f18be3..0490076f 100644 --- a/webui/static/sync-services.js +++ b/webui/static/sync-services.js @@ -4022,12 +4022,18 @@ async function handleDbUpdateButtonClick() { if (currentAction === 'Update Database') { const refreshSelect = document.getElementById('db-refresh-type'); - const isFullRefresh = refreshSelect.value === 'full'; + const refreshType = refreshSelect.value; + const isFullRefresh = refreshType === 'full'; + const isDeepScan = refreshType === 'deep'; if (isFullRefresh) { // Replicates the QMessageBox confirmation from the GUI const confirmed = await showConfirmDialog({ title: 'Full Refresh', message: 'This will clear and rebuild the database for the active server. It can take a long time.\n\nAre you sure you want to proceed?', confirmText: 'Proceed' }); if (!confirmed) return; + } else if (isDeepScan) { + // Deep scan removes stale entries — confirm like the dashboard deep scan does. + const confirmed = await showConfirmDialog({ title: 'Deep Scan', message: 'A deep scan re-checks every track in your media server library against the database.\n\n• Adds any new tracks that were missed\n• Removes tracks no longer on your server\n• Preserves all existing metadata and enrichment data\n\nThis may take a while for large libraries. Proceed?', confirmText: 'Proceed' }); + if (!confirmed) return; } try { @@ -4036,7 +4042,9 @@ async function handleDbUpdateButtonClick() { const response = await fetch('/api/database/update', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ full_refresh: isFullRefresh }) + // Deep scan and full refresh use distinct backend flags; deep + // scan takes precedence server-side, so send only its flag. + body: JSON.stringify(isDeepScan ? { deep_scan: true } : { full_refresh: isFullRefresh }) }); if (response.ok) { From a42f8ecc10b047ab46cb62c1abc3e6cde09df348 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 11:07:09 -0700 Subject: [PATCH 35/52] UI: move Downloads above Automations in the sidebar Reorder the sidebar nav so Downloads sits between Wishlist and Automations. Mobile nav reuses the same .nav-button elements and the helper/onboarding references are selector-keyed, so no other changes are needed. --- webui/index.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/webui/index.html b/webui/index.html index b37d6349..301a771e 100644 --- a/webui/index.html +++ b/webui/index.html @@ -225,15 +225,15 @@ Wishlist - - - Automations - Downloads + + + Automations + Import From ae0968e1b016e8d84af0fdee944b5fa638edb1a2 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 11:18:19 -0700 Subject: [PATCH 36/52] UI: migrate watchlist/wishlist action buttons to the shared .btn primitive The watchlist + wishlist header/overview buttons used a bespoke .watchlist-action-btn family (different padding/radius/font and white primary text) instead of the shared .btn design-system primitive. Migrate all 11 of them to .btn / .btn--primary / .btn--secondary / .btn--danger so they match the rest of the app, and drop the now-dead CSS. The .watchlist-batch-remove-btn / .wishlist-batch-remove-btn hook classes are kept on the remove buttons (their !important red overrides compose correctly over .btn--secondary). Static HTML only; no JS-injected usages, and mobile.css overrides target .playlist-modal-btn, not these. --- webui/index.html | 22 +++++++-------- webui/static/style.css | 63 +++--------------------------------------- 2 files changed, 15 insertions(+), 70 deletions(-) diff --git a/webui/index.html b/webui/index.html index 301a771e..c26c510a 100644 --- a/webui/index.html +++ b/webui/index.html @@ -6822,19 +6822,19 @@
- - - - @@ -6874,7 +6874,7 @@ Select All -
@@ -6891,7 +6891,7 @@

Your watchlist is empty

Use Search to find an artist, then add them to your watchlist from the artist page.

- +
@@ -6914,11 +6914,11 @@
- - @@ -6948,7 +6948,7 @@
- @@ -6966,14 +6966,14 @@
-
diff --git a/webui/static/style.css b/webui/static/style.css index 2603908f..f0f14cf4 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -59728,65 +59728,10 @@ body.reduce-effects *::after { padding: 0 4px; } -.watchlist-action-btn { - display: inline-flex; - align-items: center; - gap: 7px; - padding: 9px 18px; - border-radius: 10px; - border: 1px solid transparent; - cursor: pointer; - font-size: 13px; - font-weight: 500; - transition: all 0.25s cubic-bezier(0.4, 0, 0.2, 1); - white-space: nowrap; - letter-spacing: 0.1px; -} - -.watchlist-action-primary { - background: linear-gradient(135deg, rgb(var(--accent-rgb)), rgba(var(--accent-rgb), 0.85)); - color: #fff; - border-color: rgba(var(--accent-rgb), 0.3); - box-shadow: 0 2px 8px rgba(var(--accent-rgb), 0.2); -} - -.watchlist-action-primary:hover { - transform: translateY(-1px); - box-shadow: 0 4px 14px rgba(var(--accent-rgb), 0.3); - filter: brightness(1.1); -} - -.watchlist-action-primary:disabled { - opacity: 0.45; - cursor: not-allowed; - transform: none; - box-shadow: none; -} - -.watchlist-action-secondary { - background: rgba(255, 255, 255, 0.05); - color: rgba(255, 255, 255, 0.7); - border-color: rgba(255, 255, 255, 0.08); -} - -.watchlist-action-secondary:hover { - background: rgba(255, 255, 255, 0.1); - color: #fff; - border-color: rgba(255, 255, 255, 0.15); - transform: translateY(-1px); -} - -.watchlist-action-danger { - background: rgba(239, 68, 68, 0.1); - color: #f87171; - border-color: rgba(239, 68, 68, 0.15); -} - -.watchlist-action-danger:hover { - background: rgba(239, 68, 68, 0.18); - border-color: rgba(239, 68, 68, 0.25); - transform: translateY(-1px); -} +/* .watchlist-action-btn* migrated to the shared .btn / .btn--primary / + .btn--secondary / .btn--danger primitive (watchlist + wishlist headers). + The .watchlist-batch-remove-btn / .wishlist-batch-remove-btn hook classes + are kept on those buttons for their own overrides. */ /* ── Scan status card ── */ .watchlist-page-scan-status { From 169c30fd5b73203bb3a03927d8aaf53c8dc38e1b Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 11:30:13 -0700 Subject: [PATCH 37/52] UI: add .btn--sm/.btn--block/.btn--warning tier; migrate sync-history buttons Formalize the compact 'toolbar' button tier as design-system modifiers (.btn--sm), plus a full-width (.btn--block) and amber caution (.btn--warning) modifier, so the many smaller per-page buttons can share the .btn primitive without being forced to the large default size. First adopter: the Sync page header buttons (.sync-history-btn) now use .btn .btn--sm .btn--secondary. The class is kept as a JS/onboarding selector hook; .auto-sync-manager-btn still tints Auto-Sync accent. --- webui/index.html | 6 +++--- webui/static/style.css | 33 +++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/webui/index.html b/webui/index.html index c26c510a..43581f33 100644 --- a/webui/index.html +++ b/webui/index.html @@ -986,9 +986,9 @@ server

- - - + + +
diff --git a/webui/static/style.css b/webui/static/style.css index f0f14cf4..11f64865 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -8059,6 +8059,30 @@ body.helper-mode-active #dashboard-activity-feed:hover { box-shadow: 0 8px 24px rgba(29, 185, 84, 0.4); } +/* Amber "caution" color (e.g. bulk fix-all). */ +.btn--warning { + background: rgba(245, 158, 11, 0.12); + color: #fbbf24; + border: 1px solid rgba(245, 158, 11, 0.3); +} +.btn--warning:hover:not(:disabled) { + background: rgba(245, 158, 11, 0.22); + border-color: rgba(245, 158, 11, 0.45); +} + +/* Compact toolbar size — the second button tier used in headers, bulk-action + rows and toolbars (smaller than the default large action button). Combine + with a color modifier, e.g. `.btn .btn--sm .btn--secondary`. */ +.btn--sm { + padding: 6px 14px; + font-size: 12px; + border-radius: 8px; + gap: 6px; +} + +/* Full-width (form submit buttons, etc.). */ +.btn--block { width: 100%; } + /* Shared loading state (JS toggles `.loading`). */ .btn.loading { background: rgba(255, 193, 7, 0.9); @@ -11298,12 +11322,9 @@ body.helper-mode-active #dashboard-activity-feed:hover { .sync-history-page-btn:hover:not(:disabled) { color: rgb(var(--accent-rgb)); background: rgba(var(--accent-rgb), 0.1); } .sync-history-page-btn:disabled { opacity: 0.3; cursor: default; } .sync-history-page-info { font-size: 12px; color: rgba(255, 255, 255, 0.35); } -.sync-history-btn { - font-size: 12px; font-weight: 500; color: rgba(255, 255, 255, 0.5); - background: rgba(255, 255, 255, 0.05); border: 1px solid rgba(255, 255, 255, 0.08); - border-radius: 6px; padding: 5px 12px; cursor: pointer; transition: all 0.2s ease; -} -.sync-history-btn:hover { color: rgb(var(--accent-rgb)); background: rgba(var(--accent-rgb), 0.1); border-color: rgba(var(--accent-rgb), 0.3); } +/* .sync-history-btn styling migrated to .btn .btn--sm .btn--secondary. + The class is kept on the markup as a JS/onboarding selector hook, and + .auto-sync-manager-btn still tints the Auto-Sync button accent. */ .auto-sync-manager-btn { color: rgb(var(--accent-light-rgb)); From 9b34d06b6d217da1b80901fac86668c527c43abf Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 11:40:31 -0700 Subject: [PATCH 38/52] UI: migrate remaining compact button families to the .btn--sm tier Continue the design-system unification (kettui UI-consistency item): migrate the five remaining compact button families onto the shared .btn .btn--sm primitive + color modifiers, and drop their bespoke base CSS (net -125 lines of CSS). - ya-header-btn (Your Albums/Artists, discover.js-injected) -> .btn .btn--sm .btn--secondary; ya-refresh/ya-settings/ya-viewall co-modifiers kept. - explorer-action-btn (Playlist Explorer) -> .btn--secondary / .btn--primary. - repair-bulk-btn -> .btn--secondary / .btn--primary / .btn--warning (fix-all). - enhanced-bulk-btn (Library bulk bar, library.js-injected) -> .btn--primary/ --secondary/--danger; class kept as a hook for the mobile.css size override + the .tag-write / .rg-analyze special colors. - profile-create-btn (init.js-injected) -> .btn .btn--block .btn--primary; class kept for the scoped .profile-edit-buttons flex:1 rule. mini-nav-btn deliberately left as a distinct icon-button archetype. --- webui/index.html | 52 ++++++------- webui/static/discover.js | 12 +-- webui/static/init.js | 4 +- webui/static/library.js | 4 +- webui/static/style.css | 155 ++++----------------------------------- 5 files changed, 51 insertions(+), 176 deletions(-) diff --git a/webui/index.html b/webui/index.html index 43581f33..4fe0bc6b 100644 --- a/webui/index.html +++ b/webui/index.html @@ -136,13 +136,13 @@ Can download music
- +
@@ -2761,8 +2761,8 @@
- - + +
@@ -2774,10 +2774,10 @@ tracks selected
- - - - + + + +
@@ -2801,8 +2801,8 @@ Sync to server
- - + +
@@ -2829,8 +2829,8 @@ Sync to server
- - + +
@@ -2847,8 +2847,8 @@

@@ -3043,13 +3043,13 @@

Artists you follow across your music services

- - - @@ -3068,13 +3068,13 @@

Albums you've saved across your music services

- - -
@@ -3679,15 +3679,15 @@ 0 albums selected
- - - @@ -6395,9 +6395,9 @@
diff --git a/webui/static/discover.js b/webui/static/discover.js index b5303e5d..a4b28736 100644 --- a/webui/static/discover.js +++ b/webui/static/discover.js @@ -4654,15 +4654,15 @@ async function openYourArtistInfoModal(poolId) { // Footer if (footerEl) { const watchBtn = pool.on_watchlist - ? `` - : ``; + ? `` + : ``; footerEl.innerHTML = ` ${watchBtn} - - + View Discography @@ -6436,8 +6436,8 @@ function _showArtistMapSearchPrompt() {
- - + diff --git a/webui/static/init.js b/webui/static/init.js index 10a047b4..80fd8564 100644 --- a/webui/static/init.js +++ b/webui/static/init.js @@ -1829,7 +1829,7 @@ function showProfileEditForm(profileId, currentName, currentColor, currentAvatar btnRow.className = 'profile-edit-buttons'; const saveBtn = document.createElement('button'); - saveBtn.className = 'profile-create-btn'; + saveBtn.className = 'btn btn--block btn--primary profile-create-btn'; saveBtn.textContent = 'Save'; saveBtn.onclick = async () => { const newName = nameInput.value.trim(); @@ -1957,7 +1957,7 @@ function showSelfEditForm() { btnRow.style.marginTop = '12px'; const saveBtn = document.createElement('button'); - saveBtn.className = 'profile-create-btn'; + saveBtn.className = 'btn btn--block btn--primary profile-create-btn'; saveBtn.textContent = 'Save'; saveBtn.onclick = async () => { const newName = nameInput.value.trim(); diff --git a/webui/static/library.js b/webui/static/library.js index a979233c..baeeb7ec 100644 --- a/webui/static/library.js +++ b/webui/static/library.js @@ -6395,8 +6395,8 @@ function openHaveMissingTrackModal(track, album) {
Waiting to start.
- - + +
`; overlay.appendChild(modal); diff --git a/webui/static/style.css b/webui/static/style.css index 11f64865..1718d785 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -34318,13 +34318,9 @@ div.artist-hero-badge { .ya-watchlist-btn.active { color: var(--accent); border-color: rgba(var(--accent-rgb),0.3); background: rgba(var(--accent-rgb),0.15); } /* Your Artists header buttons */ -.ya-header-btn { - display: flex; align-items: center; gap: 6px; - padding: 7px 14px; border-radius: 10px; font-size: 12px; font-weight: 600; - cursor: pointer; transition: all 0.2s; border: 1px solid rgba(255,255,255,0.08); - background: rgba(255,255,255,0.04); color: rgba(255,255,255,0.5); -} -.ya-header-btn:hover { background: rgba(255,255,255,0.08); color: rgba(255,255,255,0.8); border-color: rgba(255,255,255,0.12); } +/* .ya-header-btn base styling migrated to .btn .btn--sm .btn--secondary. + Class kept on the markup (discover.js injects it) along with the + .ya-refresh-btn / .ya-settings-btn / .ya-viewall-btn co-modifiers. */ .ya-refresh-btn { padding: 7px 10px; } .ya-refresh-btn:hover svg { transform: rotate(45deg); } .ya-refresh-btn svg { transition: transform 0.3s; } @@ -43230,22 +43226,9 @@ div.artist-hero-badge { border-color: #fff; } -.profile-create-btn { - width: 100%; - padding: 10px; - background: #6366f1; - color: #fff; - border: none; - border-radius: 8px; - font-size: 14px; - font-weight: 500; - cursor: pointer; - transition: background 0.2s; -} - -.profile-create-btn:hover { - background: #4f46e5; -} +/* .profile-create-btn migrated to .btn .btn--block .btn--primary. The class + is kept on the markup (init.js sets it) as a hook for the scoped + `.profile-edit-buttons .profile-create-btn { flex: 1 }` layout rule. */ .admin-pin-section { margin-top: 20px; @@ -46445,36 +46428,11 @@ textarea.enhanced-meta-field-input { display: flex; gap: 8px; } -.enhanced-bulk-btn { - padding: 8px 16px; - border: none; - border-radius: 8px; - font-size: 12px; - font-weight: 600; - cursor: pointer; - transition: all 0.2s ease; -} -.enhanced-bulk-btn.primary { - background: rgb(var(--accent-rgb)); - color: #ffffff; -} -.enhanced-bulk-btn.primary:hover { - background: rgb(var(--accent-light-rgb)); -} -.enhanced-bulk-btn.secondary { - background: rgba(255,255,255,0.1); - color: rgba(255,255,255,0.8); -} -.enhanced-bulk-btn.secondary:hover { - background: rgba(255,255,255,0.15); -} -.enhanced-bulk-btn.clear { - background: rgba(255, 65, 54, 0.1); - color: #ff4136; -} -.enhanced-bulk-btn.clear:hover { - background: rgba(255, 65, 54, 0.2); -} +/* .enhanced-bulk-btn base + .primary/.secondary/.clear migrated to + .btn .btn--sm (.btn--primary / .btn--secondary / .btn--danger). The + .enhanced-bulk-btn class is kept on the markup as a hook for the mobile + size override (.enhanced-bulk-bar .enhanced-bulk-btn) and the special + .tag-write / .rg-analyze color variants below. */ /* Bulk Edit Modal */ .enhanced-bulk-modal { @@ -52329,42 +52287,8 @@ tr.tag-diff-same { margin-right: 4px; } -.repair-bulk-btn { - background: rgba(255, 255, 255, 0.06); - border: 1px solid rgba(255, 255, 255, 0.1); - border-radius: 6px; - color: rgba(255, 255, 255, 0.7); - padding: 6px 14px; - font-size: 12px; - cursor: pointer; - transition: all 0.2s ease; -} - -.repair-bulk-btn:hover { - background: rgba(255, 255, 255, 0.12); - color: #fff; -} - -.repair-bulk-btn.fix { - background: rgba(var(--accent-rgb, 99, 102, 241), 0.12); - border-color: rgba(var(--accent-rgb, 99, 102, 241), 0.3); - color: rgb(var(--accent-light-rgb, 129, 140, 248)); - font-weight: 600; -} - -.repair-bulk-btn.fix:hover { - background: rgba(var(--accent-rgb, 99, 102, 241), 0.2); -} - -.repair-bulk-btn.fix-all { - background: rgba(245, 158, 11, 0.12); - border-color: rgba(245, 158, 11, 0.3); - color: #fbbf24; - font-weight: 600; -} -.repair-bulk-btn.fix-all:hover { - background: rgba(245, 158, 11, 0.22); -} +/* .repair-bulk-btn / .fix / .fix-all migrated to .btn .btn--sm + (.btn--secondary / .btn--primary / .btn--warning). */ .repair-clear-btn { margin-left: auto; @@ -54910,57 +54834,8 @@ tr.tag-diff-same { gap: 6px; } -.explorer-action-btn { - padding: 7px 14px; - font-size: 11px; - font-weight: 600; - color: rgba(255, 255, 255, 0.5); - background: rgba(255, 255, 255, 0.04); - border: 1px solid rgba(255, 255, 255, 0.07); - border-radius: 8px; - cursor: pointer; - transition: all 0.2s; - letter-spacing: 0.2px; - display: flex; - align-items: center; - gap: 5px; -} -.explorer-action-btn svg { opacity: 0.5; } - -.explorer-action-btn:hover { - background: rgba(255, 255, 255, 0.08); - color: rgba(255, 255, 255, 0.85); - border-color: rgba(255, 255, 255, 0.12); -} -.explorer-action-btn:hover svg { opacity: 0.8; } - -.explorer-action-btn.primary { - background: linear-gradient(135deg, var(--accent), color-mix(in srgb, var(--accent) 80%, #fff)); - border-color: transparent; - color: #fff; - font-weight: 700; - padding: 8px 20px; - font-size: 12px; - position: relative; - overflow: hidden; - box-shadow: 0 2px 10px rgba(var(--accent-rgb), 0.2); -} -.explorer-action-btn.primary svg { opacity: 1; } - -.explorer-action-btn.primary::after { - content: ''; - position: absolute; - inset: 0; - background: linear-gradient(135deg, rgba(255, 255, 255, 0.15) 0%, transparent 50%); - border-radius: inherit; - pointer-events: none; -} - -.explorer-action-btn.primary:hover { - box-shadow: 0 6px 22px rgba(var(--accent-rgb), 0.4); - filter: brightness(1.1); - transform: translateY(-1px); -} +/* .explorer-action-btn / .primary migrated to .btn .btn--sm + (.btn--secondary / .btn--primary). */ /* Viewport */ .explorer-viewport { From 2bb935b9d786f039f007bc520ad339cb58988044 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 12:04:11 -0700 Subject: [PATCH 39/52] DB: stop watchlist_artists rebuilds from dropping amazon_artist_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit amazon_artist_id is added to watchlist_artists via ALTER (music_database.py ~1732), but both table-rebuild migrations — the spotify_id-nullable fix (_fix_watchlist_spotify_id_nullable, two CREATE variants) and the profile-scoped UNIQUE rebuild — recreated the table from a hardcoded column list that omitted amazon_artist_id. Because shared_cols filters new_cols against the old table, the column and any stored Amazon artist IDs were silently dropped on every init (fresh OR upgraded), so Amazon watchlist IDs never persisted at all. Fix: add amazon_artist_id to all three rebuild CREATE schemas, both rebuild new_cols lists, and the base CREATE TABLE (so fresh installs are consistent and don't rely on the ALTER). Purely additive, column-named inserts + Row factory mean column position is irrelevant. Tests (tests/test_db_watchlist_amazon_id_migration.py): drive the real migrations via MusicDatabase() against a seeded pre-migration temp DB and assert the column + data survive; differential-proven to FAIL pre-fix. --- database/music_database.py | 10 +- .../test_db_watchlist_amazon_id_migration.py | 120 ++++++++++++++++++ 2 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 tests/test_db_watchlist_amazon_id_migration.py diff --git a/database/music_database.py b/database/music_database.py index bc08e179..ebba3893 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -337,6 +337,7 @@ class MusicDatabase: deezer_artist_id TEXT, discogs_artist_id TEXT, musicbrainz_artist_id TEXT, + amazon_artist_id TEXT, artist_name TEXT NOT NULL, date_added TIMESTAMP DEFAULT CURRENT_TIMESTAMP, last_scan_timestamp TIMESTAMP, @@ -1826,6 +1827,7 @@ class MusicDatabase: deezer_artist_id TEXT, discogs_artist_id TEXT, musicbrainz_artist_id TEXT, + amazon_artist_id TEXT, profile_id INTEGER DEFAULT 1, UNIQUE(profile_id, spotify_artist_id), UNIQUE(profile_id, itunes_artist_id) @@ -1854,7 +1856,8 @@ class MusicDatabase: itunes_artist_id TEXT, deezer_artist_id TEXT, discogs_artist_id TEXT, - musicbrainz_artist_id TEXT + musicbrainz_artist_id TEXT, + amazon_artist_id TEXT ) """) @@ -1867,7 +1870,7 @@ class MusicDatabase: 'include_remixes', 'include_acoustic', 'include_compilations', 'include_instrumentals', 'lookback_days', 'itunes_artist_id', 'deezer_artist_id', 'discogs_artist_id', - 'musicbrainz_artist_id', 'profile_id'] + 'musicbrainz_artist_id', 'amazon_artist_id', 'profile_id'] shared_cols = [c for c in new_cols if c in old_cols] cols_str = ', '.join(shared_cols) cursor.execute(f"INSERT INTO watchlist_artists_new ({cols_str}) SELECT {cols_str} FROM watchlist_artists") @@ -2711,6 +2714,7 @@ class MusicDatabase: deezer_artist_id TEXT, discogs_artist_id TEXT, musicbrainz_artist_id TEXT, + amazon_artist_id TEXT, profile_id INTEGER DEFAULT 1, UNIQUE(profile_id, spotify_artist_id), UNIQUE(profile_id, itunes_artist_id) @@ -2724,7 +2728,7 @@ class MusicDatabase: 'include_remixes', 'include_acoustic', 'include_compilations', 'include_instrumentals', 'lookback_days', 'itunes_artist_id', 'deezer_artist_id', 'discogs_artist_id', - 'musicbrainz_artist_id', 'profile_id'] + 'musicbrainz_artist_id', 'amazon_artist_id', 'profile_id'] shared_cols = [c for c in new_cols if c in col_names] cols_str = ', '.join(shared_cols) diff --git a/tests/test_db_watchlist_amazon_id_migration.py b/tests/test_db_watchlist_amazon_id_migration.py new file mode 100644 index 00000000..cd017f84 --- /dev/null +++ b/tests/test_db_watchlist_amazon_id_migration.py @@ -0,0 +1,120 @@ +"""Regression for the watchlist_artists rebuild dropping amazon_artist_id. + +`amazon_artist_id` is added to watchlist_artists via ALTER (music_database.py +~1732), but the table-rebuild migrations (the spotify_id-nullable fix and the +profile-scoped UNIQUE rebuild) recreated the table from a hardcoded column list +that omitted amazon_artist_id — so on upgrade the column AND any stored Amazon +artist IDs were silently dropped. + +These tests drive the REAL migrations through MusicDatabase() against a fresh +temp database that starts in the pre-migration shape (no profile-scoped UNIQUE, +amazon_artist_id present with data), then assert the column and its data survive. + +Proven differential: with database/music_database.py reverted to pre-fix, +test_amazon_artist_id_data_survives_rebuild FAILS (the column/data are dropped); +with the fix it passes. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from database.music_database import MusicDatabase + + +# A pre-profile-migration watchlist_artists schema (no UNIQUE(profile_id, ...), +# i.e. exactly the state that triggers the rebuild path) that ALREADY carries +# amazon_artist_id + the other source-id columns — mirroring a real DB that ran +# the 1732 ALTER before the rebuild migrations existed. +_OLD_WATCHLIST_SCHEMA = """ + CREATE TABLE watchlist_artists ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + spotify_artist_id TEXT UNIQUE, + itunes_artist_id TEXT, + deezer_artist_id TEXT, + discogs_artist_id TEXT, + musicbrainz_artist_id TEXT, + amazon_artist_id TEXT, + artist_name TEXT NOT NULL, + date_added TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + last_scan_timestamp TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) +""" + + +def _seed_old_db(db_path: Path) -> None: + """Create a pre-migration watchlist_artists with an Amazon-tagged row.""" + conn = sqlite3.connect(str(db_path)) + try: + conn.execute(_OLD_WATCHLIST_SCHEMA) + conn.execute( + "INSERT INTO watchlist_artists " + "(spotify_artist_id, amazon_artist_id, artist_name) VALUES (?, ?, ?)", + ("spfy_abc", "B0AMAZONXYZ", "Amazon Tagged Artist"), + ) + conn.commit() + finally: + conn.close() + + +def _watchlist_columns(db_path: Path) -> list: + conn = sqlite3.connect(str(db_path)) + try: + return [r[1] for r in conn.execute("PRAGMA table_info(watchlist_artists)")] + finally: + conn.close() + + +def test_amazon_artist_id_column_survives_rebuild(tmp_path: Path) -> None: + """After the real migrations run, watchlist_artists must still have the + amazon_artist_id column (the rebuild must not drop it).""" + db_path = tmp_path / "old_library.db" + _seed_old_db(db_path) + + # Driving MusicDatabase against this path runs the real _initialize_database, + # which fires the watchlist_artists rebuild(s). + MusicDatabase(str(db_path)) + + cols = _watchlist_columns(db_path) + assert "amazon_artist_id" in cols, ( + "amazon_artist_id was dropped by the watchlist_artists rebuild; " + f"columns are: {cols}" + ) + # The rebuild's whole purpose: profile-scoped uniqueness must still apply. + assert "profile_id" in cols + + +def test_amazon_artist_id_data_survives_rebuild(tmp_path: Path) -> None: + """The stored Amazon artist ID must be carried across the rebuild, not lost. + This is the test that FAILS on pre-fix code.""" + db_path = tmp_path / "old_library.db" + _seed_old_db(db_path) + + MusicDatabase(str(db_path)) + + conn = sqlite3.connect(str(db_path)) + try: + row = conn.execute( + "SELECT amazon_artist_id FROM watchlist_artists WHERE artist_name = ?", + ("Amazon Tagged Artist",), + ).fetchone() + finally: + conn.close() + + assert row is not None, "the watchlist row was lost entirely during rebuild" + assert row[0] == "B0AMAZONXYZ", ( + f"amazon_artist_id data was not preserved across rebuild (got {row[0]!r})" + ) + + +def test_fresh_db_has_amazon_artist_id_column(tmp_path: Path) -> None: + """A brand-new database (no pre-existing table) must also end up with the + amazon_artist_id column, so fresh installs match upgraded ones.""" + db_path = tmp_path / "fresh_library.db" + MusicDatabase(str(db_path)) + assert "amazon_artist_id" in _watchlist_columns(db_path) From c5b02c002641e8714b92e29c6394e5fbdb106e5c Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 12:11:09 -0700 Subject: [PATCH 40/52] DB: normalize legacy comma-separated genres to canonical JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit artists.genres / albums.genres stored EITHER a JSON array (new writes) OR a legacy comma-separated string (old writes), forcing every reader to try-JSON-then-split. Add a marker-gated one-time migration (_normalize_genres_to_json) that rewrites legacy rows to JSON in place, mirroring the readers' exact parse (JSON list, else comma-split/strip/ drop-empties) so genre VALUES are unchanged — only the storage format. Per-row diffed (already-canonical rows untouched, no churn) and non-fatal on error, consistent with the other migrations. Readers still tolerate both formats, so this breaks nothing; it just removes the dual-format debt. Tests: tests/test_db_genres_json_normalization.py — CSV->JSON, JSON-unchanged, whitespace/empties dropped, albums table, legacy-reader-equivalence, idempotent re-run, marker set on fresh init. --- database/music_database.py | 52 +++++++++ tests/test_db_genres_json_normalization.py | 120 +++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 tests/test_db_genres_json_normalization.py diff --git a/database/music_database.py b/database/music_database.py index ebba3893..523d7cdf 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -793,6 +793,7 @@ class MusicDatabase: logger.error(f"Personalized-playlist schema init failed: {ps_err}") self._ensure_core_media_schema_columns(cursor) + self._normalize_genres_to_json(cursor) conn.commit() logger.info("Database initialized successfully") @@ -803,6 +804,57 @@ class MusicDatabase: self._init_manual_library_match_table() + def _normalize_genres_to_json(self, cursor): + """One-time: rewrite legacy comma-separated genres to canonical JSON arrays. + + ``artists.genres`` / ``albums.genres`` historically stored EITHER a JSON + array (new writes) OR a comma-separated string (old writes), so every + reader has to try-JSON-then-split. This normalizes existing rows to JSON + in place. It mirrors the readers' exact parse (JSON list, else + comma-split/strip/drop-empties), so the genre VALUES are unchanged — only + the storage format. Marker-gated to run once, and per-row diffed so a + re-run (or a row already in JSON form) is a no-op. Non-fatal on error, + like the other migrations. + """ + import json + try: + cursor.execute("SELECT value FROM metadata WHERE key = 'genres_json_normalized' LIMIT 1") + if cursor.fetchone(): + return + + def _to_list(raw): + # Identical semantics to the genres readers elsewhere in this file. + try: + parsed = json.loads(raw) + return parsed if isinstance(parsed, list) else [str(parsed)] + except (json.JSONDecodeError, ValueError, TypeError): + return [g.strip() for g in raw.split(',') if g.strip()] + + total = 0 + for table in ('artists', 'albums'): + cursor.execute( + f"SELECT id, genres FROM {table} " + f"WHERE genres IS NOT NULL AND TRIM(genres) != ''" + ) + pending = [] + for row in cursor.fetchall(): + rid, raw = row[0], row[1] + canonical = json.dumps(_to_list(raw)) + if canonical != raw: # leave already-canonical rows untouched + pending.append((canonical, rid)) + for canonical, rid in pending: + cursor.execute(f"UPDATE {table} SET genres = ? WHERE id = ?", (canonical, rid)) + total += 1 + + cursor.execute( + "INSERT OR REPLACE INTO metadata (key, value, updated_at) " + "VALUES ('genres_json_normalized', 'true', CURRENT_TIMESTAMP)" + ) + if total: + logger.info("Normalized %d legacy genres value(s) to JSON", total) + except Exception as e: + logger.error(f"Error normalizing genres to JSON: {e}") + def _ensure_core_media_schema_columns(self, cursor): """Repair required media-library columns that older migrations may miss. diff --git a/tests/test_db_genres_json_normalization.py b/tests/test_db_genres_json_normalization.py new file mode 100644 index 00000000..078ab2a7 --- /dev/null +++ b/tests/test_db_genres_json_normalization.py @@ -0,0 +1,120 @@ +"""Tests for the one-time genres CSV->JSON normalization migration. + +artists.genres / albums.genres historically stored either a JSON array (new +writes) or a legacy comma-separated string (old writes). _normalize_genres_to_json +rewrites legacy rows to canonical JSON, mirroring the readers' exact parse so the +genre VALUES are unchanged — only the storage format. These tests drive the real +method on a temp database. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from database.music_database import MusicDatabase + + +def _fresh_db(tmp_path: Path) -> MusicDatabase: + # Init creates the schema and (harmlessly) runs the normalization once on the + # empty DB, setting the marker. Tests clear the marker + seed, then call the + # method directly so they exercise the real normalization logic. + return MusicDatabase(str(tmp_path / "library.db")) + + +def _seed_and_normalize(db: MusicDatabase, artists, albums=()): + """Insert (id, name, genres) artists and (id, artist_id, title, genres) albums + with the marker cleared, then run the real migration. Returns nothing.""" + with db._get_connection() as conn: + cur = conn.cursor() + cur.execute("DELETE FROM metadata WHERE key = 'genres_json_normalized'") + for aid, name, genres in artists: + cur.execute( + "INSERT INTO artists (id, name, genres) VALUES (?, ?, ?)", + (aid, name, genres), + ) + for alid, artist_id, title, genres in albums: + cur.execute( + "INSERT INTO albums (id, artist_id, title, genres) VALUES (?, ?, ?, ?)", + (alid, artist_id, title, genres), + ) + conn.commit() + db._normalize_genres_to_json(cur) + conn.commit() + + +def _get_genres(db: MusicDatabase, table: str, rid: str): + with db._get_connection() as conn: + row = conn.execute(f"SELECT genres FROM {table} WHERE id = ?", (rid,)).fetchone() + return row[0] + + +def test_csv_genres_normalized_to_json(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + _seed_and_normalize(db, [("a1", "Artist One", "Rock, Pop, Jazz")]) + stored = _get_genres(db, "artists", "a1") + assert json.loads(stored) == ["Rock", "Pop", "Jazz"] + + +def test_existing_json_genres_left_unchanged(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + canonical = json.dumps(["Hip-Hop", "Soul"]) + _seed_and_normalize(db, [("a1", "Artist One", canonical)]) + # Byte-for-byte identical — no needless churn on already-canonical rows. + assert _get_genres(db, "artists", "a1") == canonical + + +def test_single_genre_without_comma(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + _seed_and_normalize(db, [("a1", "Artist One", "Electronic")]) + assert json.loads(_get_genres(db, "artists", "a1")) == ["Electronic"] + + +def test_csv_whitespace_and_empties_dropped(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + _seed_and_normalize(db, [("a1", "Artist One", " Rock ,, Pop , ")]) + assert json.loads(_get_genres(db, "artists", "a1")) == ["Rock", "Pop"] + + +def test_albums_table_also_normalized(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + _seed_and_normalize( + db, + artists=[("a1", "Artist One", "Rock")], + albums=[("al1", "a1", "Album One", "Soul, Funk")], + ) + assert json.loads(_get_genres(db, "albums", "al1")) == ["Soul", "Funk"] + + +def test_values_match_legacy_reader_semantics(tmp_path: Path) -> None: + """The normalized list must equal what the legacy CSV reader would produce, + so downstream genre values are identical pre- and post-migration.""" + db = _fresh_db(tmp_path) + raw = "Rock, Pop, Hip-Hop/Rap" + _seed_and_normalize(db, [("a1", "Artist One", raw)]) + legacy = [g.strip() for g in raw.split(",") if g.strip()] + assert json.loads(_get_genres(db, "artists", "a1")) == legacy + + +def test_idempotent_rerun(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + _seed_and_normalize(db, [("a1", "Artist One", "Rock, Pop")]) + first = _get_genres(db, "artists", "a1") + # Marker is now set; a second run must be a no-op and leave the value identical. + with db._get_connection() as conn: + cur = conn.cursor() + db._normalize_genres_to_json(cur) + conn.commit() + assert _get_genres(db, "artists", "a1") == first + assert json.loads(first) == ["Rock", "Pop"] + + +def test_marker_set_after_fresh_init(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + with db._get_connection() as conn: + row = conn.execute( + "SELECT value FROM metadata WHERE key = 'genres_json_normalized'" + ).fetchone() + assert row is not None and row[0] == "true" From b55faff54b8c3121d148589e26c6eb9e37931d4a Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 12:14:20 -0700 Subject: [PATCH 41/52] DB: add schema_migrations ledger + PRAGMA user_version backstop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migration state was scattered across PRAGMA-table_info guards, sentinel marker tables (_genius_search_fix_applied, ...) and metadata-flag rows (id_columns_migrated, ...), with no single source of truth and no schema version — so a half-migrated DB was undetectable. Add a non-gating backstop: a schema_migrations(name, applied_at) ledger plus a _sync_migration_ledger pass (runs last in init) that back-fills the ledger from the existing signals and stamps PRAGMA user_version. ADDITIVE only — existing migrations keep their own idempotency gates; nothing decides whether a migration runs based on the ledger or the version. New one-time migrations call _record_migration (the genres migration already does). Tests: tests/test_db_migration_ledger.py — table exists, user_version stamped, record idempotent, genres recorded on fresh init, backfill from flag + marker, absent signals not recorded. --- database/music_database.py | 81 ++++++++++++++++++++++++++ tests/test_db_migration_ledger.py | 96 +++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 tests/test_db_migration_ledger.py diff --git a/database/music_database.py b/database/music_database.py index 523d7cdf..1394719f 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -312,6 +312,19 @@ class MusicDatabase: updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """) + + # Migration ledger — a single, readable record of which one-time + # migrations have run. ADDITIVE backstop only: existing migrations + # keep their own idempotency gates (PRAGMA checks, marker tables, + # metadata flags); this table just unifies that scattered state so a + # half-migrated DB is detectable. Nothing GATES on it. Paired with + # PRAGMA user_version (set at the end of init). + cursor.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + name TEXT PRIMARY KEY, + applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) # Wishlist table for storing failed download tracks for retry cursor.execute(""" @@ -794,6 +807,9 @@ class MusicDatabase: self._ensure_core_media_schema_columns(cursor) self._normalize_genres_to_json(cursor) + # Unify scattered migration state into the ledger + stamp the schema + # version. Additive backstop — runs last, gates nothing. + self._sync_migration_ledger(cursor) conn.commit() logger.info("Database initialized successfully") @@ -804,6 +820,70 @@ class MusicDatabase: self._init_manual_library_match_table() + # Bump when the schema's generation meaningfully changes. Stamped into + # PRAGMA user_version as a backstop indicator; nothing GATES on it yet. + SCHEMA_VERSION = 1 + + # Maps a ledger name to the EXISTING idempotency signal that proves a + # one-time migration ran: ('table', ) or ('flag', ). Used to back-fill the ledger for DBs created before it existed. + # The ledger is a non-gating backstop, so this can grow lazily — a missing + # entry just means that migration isn't surfaced in the ledger (harmless). + _KNOWN_MIGRATION_SIGNALS = { + 'id_columns_to_text': ('flag', 'id_columns_migrated'), + 'genres_json': ('flag', 'genres_json_normalized'), + 'metadata_cache_v1': ('flag', 'metadata_cache_v1'), + 'repair_worker_v2': ('flag', 'repair_worker_v2'), + 'spotify_library_cache_v1': ('flag', 'spotify_library_cache_v1'), + 'profiles_v1': ('flag', 'profiles_migration_v1'), + 'profiles_v2': ('flag', 'profiles_migration_v2'), + 'profiles_v3': ('flag', 'profiles_migration_v3'), + 'profiles_v4': ('flag', 'profiles_migration_v4'), + 'discovery_cache_v2': ('table', '_discovery_cache_v2_migrated'), + 'deezer_cache_v2': ('table', '_deezer_cache_v2_migrated'), + 'cache_junk_artist_purged': ('table', '_cache_junk_artist_purged'), + 'genius_search_fix': ('table', '_genius_search_fix_applied'), + } + + def _record_migration(self, cursor, name): + """Record a one-time migration in the schema_migrations ledger. + + Idempotent (INSERT OR IGNORE). New one-time migrations should call this + when they complete; the ledger is a non-gating backstop, so a failure to + record never affects correctness. + """ + try: + cursor.execute( + "INSERT OR IGNORE INTO schema_migrations (name, applied_at) " + "VALUES (?, CURRENT_TIMESTAMP)", (name,) + ) + except Exception as e: + logger.debug("Could not record migration %s in ledger: %s", name, e) + + def _sync_migration_ledger(self, cursor): + """Back-fill the ledger from existing idempotency signals and stamp + PRAGMA user_version. + + ADDITIVE + non-gating: this only RECORDS state that already exists (which + marker tables / metadata flags are present); it never decides whether a + migration runs. Safe to run on every startup. + """ + try: + cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") + tables = {r[0] for r in cursor.fetchall()} + for ledger_name, (kind, signal) in self._KNOWN_MIGRATION_SIGNALS.items(): + if kind == 'table': + present = signal in tables + else: # 'flag' — a metadata row + cursor.execute("SELECT 1 FROM metadata WHERE key = ? LIMIT 1", (signal,)) + present = cursor.fetchone() is not None + if present: + self._record_migration(cursor, ledger_name) + # Backstop version stamp; nothing gates on it. + cursor.execute(f"PRAGMA user_version = {int(self.SCHEMA_VERSION)}") + except Exception as e: + logger.error(f"Error syncing migration ledger: {e}") + def _normalize_genres_to_json(self, cursor): """One-time: rewrite legacy comma-separated genres to canonical JSON arrays. @@ -850,6 +930,7 @@ class MusicDatabase: "INSERT OR REPLACE INTO metadata (key, value, updated_at) " "VALUES ('genres_json_normalized', 'true', CURRENT_TIMESTAMP)" ) + self._record_migration(cursor, 'genres_json') if total: logger.info("Normalized %d legacy genres value(s) to JSON", total) except Exception as e: diff --git a/tests/test_db_migration_ledger.py b/tests/test_db_migration_ledger.py new file mode 100644 index 00000000..07b2d419 --- /dev/null +++ b/tests/test_db_migration_ledger.py @@ -0,0 +1,96 @@ +"""Tests for the additive schema_migrations ledger + PRAGMA user_version backstop. + +The ledger unifies the previously-scattered migration state (marker tables + +metadata flags) into one readable place so a half-migrated DB is detectable. +It is NON-GATING: nothing decides whether a migration runs based on it. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from database.music_database import MusicDatabase + + +def _fresh_db(tmp_path: Path) -> MusicDatabase: + return MusicDatabase(str(tmp_path / "library.db")) + + +def test_schema_migrations_table_exists(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + with db._get_connection() as conn: + row = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='schema_migrations'" + ).fetchone() + assert row is not None + + +def test_user_version_stamped(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + with db._get_connection() as conn: + version = conn.execute("PRAGMA user_version").fetchone()[0] + assert version == MusicDatabase.SCHEMA_VERSION == 1 + + +def test_record_migration_is_idempotent(tmp_path: Path) -> None: + db = _fresh_db(tmp_path) + with db._get_connection() as conn: + cur = conn.cursor() + db._record_migration(cur, "unit_test_mig") + db._record_migration(cur, "unit_test_mig") + conn.commit() + n = cur.execute( + "SELECT COUNT(*) FROM schema_migrations WHERE name = 'unit_test_mig'" + ).fetchone()[0] + assert n == 1 + + +def test_genres_migration_recorded_on_fresh_init(tmp_path: Path) -> None: + """The forward pattern: the genres migration records itself in the ledger.""" + db = _fresh_db(tmp_path) + with db._get_connection() as conn: + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE name = 'genres_json'" + ).fetchone() + assert row is not None + + +def test_ledger_backfills_from_existing_signals(tmp_path: Path) -> None: + """Back-fill records both metadata-flag and marker-table migrations that are + already present, under their canonical ledger names.""" + db = _fresh_db(tmp_path) + with db._get_connection() as conn: + cur = conn.cursor() + cur.execute("DELETE FROM schema_migrations") + # A metadata-flag-style signal and a marker-table-style signal. + cur.execute( + "INSERT OR REPLACE INTO metadata (key, value, updated_at) " + "VALUES ('metadata_cache_v1', '1', CURRENT_TIMESTAMP)" + ) + cur.execute( + "CREATE TABLE IF NOT EXISTS _genius_search_fix_applied " + "(applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)" + ) + conn.commit() + db._sync_migration_ledger(cur) + conn.commit() + names = {r[0] for r in cur.execute("SELECT name FROM schema_migrations")} + assert "metadata_cache_v1" in names # from the metadata flag + assert "genius_search_fix" in names # from the marker table + + +def test_ledger_does_not_record_absent_signals(tmp_path: Path) -> None: + """A migration whose signal is absent must NOT be recorded as applied.""" + db = _fresh_db(tmp_path) + with db._get_connection() as conn: + cur = conn.cursor() + cur.execute("DELETE FROM schema_migrations") + # Ensure the deezer-cache marker table does not exist. + cur.execute("DROP TABLE IF EXISTS _deezer_cache_v2_migrated") + conn.commit() + db._sync_migration_ledger(cur) + conn.commit() + names = {r[0] for r in cur.execute("SELECT name FROM schema_migrations")} + assert "deezer_cache_v2" not in names From 4fcc461616442d3b622d5585dfc0ecc04df959a7 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 12:19:59 -0700 Subject: [PATCH 42/52] Source IDs: add canonical registry, adopt at the highest-value sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same provider ID is stored under inconsistent column names across tables (deezer_id vs deezer_artist_id vs album_deezer_id vs similar_artist_deezer_id; spotify/itunes keep an entity qualifier, others don't; musicbrainz uses three nouns), so code checks 2-5 name variants everywhere. Add core/source_ids.py as the single source of truth for (provider, entity) -> column, with accessors that read an ID from a dict/sqlite3.Row robustly (canonical column first, then known aliases). NO database columns are renamed — these are the real names today; the registry just centralizes the knowledge. Targeted adoption (behavior-identical, verified): - core/artist_source_lookup.SOURCE_ID_FIELD now derives from the registry instead of duplicating the mapping (values unchanged). - web_server.py artist-detail builds artist_source_ids via source_id_map(...) instead of a hand-rolled per-source .get() dict. Broader call-site adoption deferred as clearly-scoped follow-up. Tests: tests/test_source_ids_registry.py (canonical columns, alias fallback, canonical-preferred, sqlite3.Row, source_id_map, SOURCE_ID_FIELD unchanged). Existing artist_source_lookup + artist_full_detail suites still green. --- core/artist_source_lookup.py | 16 ++-- core/source_ids.py | 144 ++++++++++++++++++++++++++++++ tests/test_source_ids_registry.py | 99 ++++++++++++++++++++ web_server.py | 16 ++-- 4 files changed, 260 insertions(+), 15 deletions(-) create mode 100644 core/source_ids.py create mode 100644 tests/test_source_ids_registry.py diff --git a/core/artist_source_lookup.py b/core/artist_source_lookup.py index 507d09e6..60fff775 100644 --- a/core/artist_source_lookup.py +++ b/core/artist_source_lookup.py @@ -21,6 +21,8 @@ from __future__ import annotations import logging from typing import Optional +from core.source_ids import id_column as _artist_id_column + logger = logging.getLogger("artist_source_lookup") @@ -29,14 +31,14 @@ SOURCE_ONLY_ARTIST_SOURCES = frozenset({ }) +# The per-source column on the ``artists`` table, derived from the canonical +# source-ID registry (the single source of truth). Values are unchanged from the +# previous hardcoded map — this just stops duplicating that knowledge here. SOURCE_ID_FIELD = { - "spotify": "spotify_artist_id", - "itunes": "itunes_artist_id", - "deezer": "deezer_id", - "discogs": "discogs_id", - "hydrabase": "soul_id", - "musicbrainz": "musicbrainz_id", - "amazon": "amazon_id", + source: _artist_id_column(source, "artist") + for source in ( + "spotify", "itunes", "deezer", "discogs", "hydrabase", "musicbrainz", "amazon", + ) } diff --git a/core/source_ids.py b/core/source_ids.py new file mode 100644 index 00000000..00168784 --- /dev/null +++ b/core/source_ids.py @@ -0,0 +1,144 @@ +"""Canonical registry of external/source ID column names. + +SoulSync stores each metadata provider's ID for an artist/album/track under a +column whose NAME is inconsistent across tables — e.g. Deezer's artist id is +``deezer_id`` on the ``artists`` table but ``deezer_artist_id`` on +``watchlist_artists`` and ``album_deezer_id`` / ``similar_artist_deezer_id`` on +the discovery tables. Spotify/iTunes keep an entity qualifier on the core tables +while Deezer/Amazon/Tidal/... don't, and MusicBrainz uses three different nouns. +The result is code that checks 2–5 property-name variants everywhere. + +This module is the single source of truth for "(provider, entity) → column". +It does NOT rename any database column — these ARE the real names today; the +registry just centralizes the knowledge and offers accessors that read an ID +from a dict / sqlite3.Row robustly (canonical column first, then known aliases), +so callers stop hand-rolling variant checks. +""" + +from __future__ import annotations + +from typing import Any, Dict, Iterable, Optional + +# Entity types this registry knows about. +ENTITIES = ("artist", "album", "track") + +# Canonical column name on the CORE table (artists / albums / tracks) for each +# (entity, provider). This is the name to prefer when reading/writing. +_CORE_ID_COLUMNS: Dict[str, Dict[str, str]] = { + "artist": { + "spotify": "spotify_artist_id", + "itunes": "itunes_artist_id", + "deezer": "deezer_id", + "musicbrainz": "musicbrainz_id", + "discogs": "discogs_id", + "amazon": "amazon_id", + "tidal": "tidal_id", + "qobuz": "qobuz_id", + "audiodb": "audiodb_id", + "genius": "genius_id", + "hydrabase": "soul_id", + }, + "album": { + "spotify": "spotify_album_id", + "itunes": "itunes_album_id", + "deezer": "deezer_id", + "musicbrainz": "musicbrainz_release_id", + "discogs": "discogs_id", + "amazon": "amazon_id", + "tidal": "tidal_id", + "qobuz": "qobuz_id", + "audiodb": "audiodb_id", + "hydrabase": "soul_id", + }, + "track": { + "spotify": "spotify_track_id", + "itunes": "itunes_track_id", + "deezer": "deezer_id", + "musicbrainz": "musicbrainz_recording_id", + "amazon": "amazon_id", + "tidal": "tidal_id", + "qobuz": "qobuz_id", + "audiodb": "audiodb_id", + "genius": "genius_id", + "hydrabase": "soul_id", + }, +} + +# Other column / dict-key names the SAME (entity, provider) ID appears under +# elsewhere (satellite tables, API payloads). Accessors check the canonical +# column first, then these, so a read works regardless of where the row/dict +# came from. Keyed by (entity, provider). +_ALIASES: Dict[tuple, tuple] = { + ("artist", "spotify"): ("similar_artist_spotify_id",), + ("artist", "itunes"): ("artist_itunes_id", "similar_artist_itunes_id"), + ("artist", "deezer"): ("deezer_artist_id", "artist_deezer_id", "similar_artist_deezer_id"), + ("artist", "musicbrainz"): ("musicbrainz_artist_id", "similar_artist_musicbrainz_id"), + ("artist", "discogs"): ("discogs_artist_id",), + ("artist", "amazon"): ("amazon_artist_id",), + ("album", "spotify"): ("album_spotify_id",), + ("album", "itunes"): ("album_itunes_id",), + ("album", "deezer"): ("deezer_album_id", "album_deezer_id"), + ("album", "discogs"): ("discogs_release_id",), + ("track", "deezer"): ("deezer_track_id",), +} + + +def id_column(provider: str, entity: str = "artist") -> Optional[str]: + """Canonical core-table column for this provider + entity, or None if the + provider isn't tracked for that entity.""" + return _CORE_ID_COLUMNS.get(entity, {}).get(provider) + + +def id_keys(provider: str, entity: str = "artist") -> tuple: + """All known key names (canonical first, then aliases) the ID may live + under. Useful for code that needs the full variant list explicitly.""" + keys = [] + canon = id_column(provider, entity) + if canon: + keys.append(canon) + for alias in _ALIASES.get((entity, provider), ()): # preserve order, no dups + if alias not in keys: + keys.append(alias) + return tuple(keys) + + +def _read(data: Any, key: str) -> Any: + """Read ``key`` from a dict or sqlite3.Row, returning None if absent.""" + try: + keys = data.keys() # dict and sqlite3.Row both support .keys() + except AttributeError: + return None + if key in keys: + try: + return data[key] + except (KeyError, IndexError): + return None + return None + + +def get_id(data: Any, provider: str, entity: str = "artist") -> Optional[str]: + """Read this provider's ID for ``entity`` from a dict / sqlite3.Row. + + Tries the canonical column first, then every known alias, and returns the + first non-empty value (or None). Replaces hand-rolled + ``row.get('deezer_id') or row.get('deezer_artist_id')`` chains. + """ + for key in id_keys(provider, entity): + value = _read(data, key) + if value: + return value + return None + + +def source_id_map( + data: Any, + entity: str = "artist", + providers: Optional[Iterable[str]] = None, +) -> Dict[str, Optional[str]]: + """Build a ``{provider: id}`` dict for ``entity`` from a row/dict — the + common "artist_source_ids" pattern. Defaults to every provider known for the + entity; pass ``providers`` to restrict/order the result. + """ + if providers is None: + providers = list(_CORE_ID_COLUMNS.get(entity, {}).keys()) + return {p: get_id(data, p, entity) for p in providers} diff --git a/tests/test_source_ids_registry.py b/tests/test_source_ids_registry.py new file mode 100644 index 00000000..212abf61 --- /dev/null +++ b/tests/test_source_ids_registry.py @@ -0,0 +1,99 @@ +"""Tests for the canonical source-ID registry (core/source_ids.py).""" + +from __future__ import annotations + +import sqlite3 + +import pytest + +from core import source_ids as sid + + +def test_canonical_columns_match_real_schema(): + # Spot-check the canonical names against the actual DB columns. + assert sid.id_column("spotify", "artist") == "spotify_artist_id" + assert sid.id_column("deezer", "artist") == "deezer_id" + assert sid.id_column("musicbrainz", "artist") == "musicbrainz_id" + assert sid.id_column("hydrabase", "artist") == "soul_id" + assert sid.id_column("spotify", "album") == "spotify_album_id" + assert sid.id_column("musicbrainz", "album") == "musicbrainz_release_id" + assert sid.id_column("deezer", "album") == "deezer_id" + assert sid.id_column("spotify", "track") == "spotify_track_id" + assert sid.id_column("musicbrainz", "track") == "musicbrainz_recording_id" + + +def test_id_column_unknown_returns_none(): + assert sid.id_column("nonesuch", "artist") is None + assert sid.id_column("spotify", "playlist") is None + + +def test_id_keys_canonical_first_then_aliases(): + keys = sid.id_keys("deezer", "artist") + assert keys[0] == "deezer_id" # canonical first + assert "deezer_artist_id" in keys # watchlist/pool alias + assert "similar_artist_deezer_id" in keys + + +def test_get_id_reads_canonical_column(): + row = {"deezer_id": "525046", "name": "Artist"} + assert sid.get_id(row, "deezer", "artist") == "525046" + + +def test_get_id_falls_back_to_alias(): + # A watchlist-shaped row uses deezer_artist_id, not deezer_id. + row = {"deezer_artist_id": "999", "artist_name": "X"} + assert sid.get_id(row, "deezer", "artist") == "999" + + +def test_get_id_prefers_canonical_over_alias(): + row = {"deezer_id": "canon", "deezer_artist_id": "alias"} + assert sid.get_id(row, "deezer", "artist") == "canon" + + +def test_get_id_missing_and_empty_return_none(): + assert sid.get_id({"name": "X"}, "deezer", "artist") is None + assert sid.get_id({"deezer_id": ""}, "deezer", "artist") is None + assert sid.get_id({"deezer_id": None}, "deezer", "artist") is None + + +def test_get_id_works_with_sqlite_row(): + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute("CREATE TABLE t (spotify_artist_id TEXT, name TEXT)") + conn.execute("INSERT INTO t VALUES ('spfy123', 'Artist')") + row = conn.execute("SELECT * FROM t").fetchone() + conn.close() + assert sid.get_id(row, "spotify", "artist") == "spfy123" + # A column not present on the row must not raise — just None. + assert sid.get_id(row, "deezer", "artist") is None + + +def test_source_id_map_builds_provider_dict(): + row = { + "spotify_artist_id": "s1", + "deezer_id": "d1", + "itunes_artist_id": None, + } + result = sid.source_id_map(row, "artist", providers=["spotify", "deezer", "itunes"]) + assert result == {"spotify": "s1", "deezer": "d1", "itunes": None} + + +def test_source_id_map_default_covers_all_providers(): + result = sid.source_id_map({"deezer_id": "d1"}, "artist") + assert result["deezer"] == "d1" + assert "spotify" in result and result["spotify"] is None + + +def test_source_id_field_unchanged_after_registry_refactor(): + """artist_source_lookup.SOURCE_ID_FIELD must keep its exact prior mapping + after being folded into the registry (no behavior change).""" + from core.artist_source_lookup import SOURCE_ID_FIELD + assert SOURCE_ID_FIELD == { + "spotify": "spotify_artist_id", + "itunes": "itunes_artist_id", + "deezer": "deezer_id", + "discogs": "discogs_id", + "hydrabase": "soul_id", + "musicbrainz": "musicbrainz_id", + "amazon": "amazon_id", + } diff --git a/web_server.py b/web_server.py index 27bca092..7c51d140 100644 --- a/web_server.py +++ b/web_server.py @@ -7601,15 +7601,15 @@ def get_artist_detail(artist_id): try: from core.metadata.lookup import MetadataLookupOptions from core.metadata_service import get_artist_detail_discography as _get_artist_detail_discography + from core.source_ids import source_id_map - artist_source_ids = { - 'spotify': artist_info.get('spotify_artist_id'), - 'deezer': artist_info.get('deezer_id'), - 'itunes': artist_info.get('itunes_artist_id'), - 'discogs': artist_info.get('discogs_id'), - 'hydrabase': artist_info.get('soul_id'), - 'amazon': artist_info.get('amazon_id'), - } + # Per-source artist IDs, read via the canonical source-ID registry + # (same columns as before: spotify_artist_id / deezer_id / + # itunes_artist_id / discogs_id / soul_id / amazon_id). + artist_source_ids = source_id_map( + artist_info, 'artist', + providers=('spotify', 'deezer', 'itunes', 'discogs', 'hydrabase', 'amazon'), + ) artist_detail_discography = _get_artist_detail_discography( artist_id, From 0898014364c018bf04711fc05b897ec068a4b80f Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 14:31:10 -0700 Subject: [PATCH 43/52] Fix #740: run wishlist album-bundle downloads on a dedicated pool A 2.6.3 change (c3b88e69) split the wishlist albums cycle into one batch per album. Each album batch runs an INLINE-BLOCKING soulseek/torrent/usenet album-bundle download (album_bundle_dispatch.try_dispatch -> download_album_to_staging) that holds its worker thread for the whole search+download. All of these were submitted to the shared 3-worker missing_download_executor -- the same pool used for per-track downloads AND the manual 'Download Wishlist' analysis. So a large Album-Completeness 'Fix all' (-> ~819 wishlist tracks -> ~82 per-album batches) saturated all 3 workers with blocking album downloads; the manual wishlist analysis could never get a thread ('Library Analysis' stuck on Pending), the other ~79 batches sat in phase='queued' forever, and auto-cleanup (which only evicts terminal-phase batches) never cleared them -> jam until restart. Fixing batch STATUS would not help: the threads are blocked inside the download, not waiting on a phase flip. Fix: add a dedicated bounded album_bundle_executor (max_workers=3) and route the AUTO per-album bundle batches to it, keeping the shared pool free for analysis / per-track / the manual wishlist (which always starts now). Hung/slow album downloads can only delay other album downloads, never the user-facing path. Additive and decoupled; the submit site falls back to the shared pool when the album pool isn't wired (older callers / tests) so behavior is unchanged there. The manual path is untouched (it already runs album bundles serially on a single thread, by design). Tests (tests/wishlist/test_automation.py): album sub-batches route to the dedicated pool while the residual per-track batch stays on the shared pool; fallback-to-shared-pool when no album pool is wired. Existing auto-processing tests still green (fallback preserves prior behavior). 707 passed across wishlist + downloads suites. --- core/wishlist/processing.py | 18 ++++++- tests/wishlist/test_automation.py | 85 +++++++++++++++++++++++++++++++ web_server.py | 12 +++++ 3 files changed, 114 insertions(+), 1 deletion(-) diff --git a/core/wishlist/processing.py b/core/wishlist/processing.py index 48d9126d..487401c8 100644 --- a/core/wishlist/processing.py +++ b/core/wishlist/processing.py @@ -67,6 +67,14 @@ class WishlistAutoProcessingRuntime: current_time_fn: Callable[[], float] profile_id: int = 1 logger: Any = module_logger + # Dedicated pool for the inline-blocking per-album bundle downloads. + # Album-bundle batches block their worker thread for the whole search + + # download; running them on the shared ``missing_download_executor`` lets a + # burst of album batches (e.g. a big Album-Completeness "Fix all" → wishlist) + # starve the per-track flow AND the user's manual "Download Wishlist" (#740). + # Routing them here keeps the shared pool free. Falls back to the shared + # executor when unset (older callers / tests) — see the submit site below. + album_bundle_executor: Any = None def remove_completed_tracks_from_wishlist( @@ -901,7 +909,15 @@ def process_wishlist_automatically(runtime: WishlistAutoProcessingRuntime, autom f"({len(group.tracks)} tracks) → {album_batch_id} [run {wishlist_run_id[:8]}]" ) _submitted_batches.append(album_batch_id) - runtime.missing_download_executor.submit( + # Album bundles block their worker thread for the whole + # search+download, so run them on the dedicated album + # pool — never the shared pool that serves analysis, + # per-track downloads and the manual wishlist (#740). + # Fall back to the shared pool if unset (older callers). + _album_executor = ( + runtime.album_bundle_executor or runtime.missing_download_executor + ) + _album_executor.submit( runtime.run_full_missing_tracks_process, album_batch_id, playlist_id, group.tracks, ) diff --git a/tests/wishlist/test_automation.py b/tests/wishlist/test_automation.py index f4453b8a..16ae5f74 100644 --- a/tests/wishlist/test_automation.py +++ b/tests/wishlist/test_automation.py @@ -131,6 +131,7 @@ def _build_runtime( batch_map=None, guard_acquired=True, is_actually_processing=False, + album_bundle_executor=None, ): if progress_calls is None: progress_calls = [] @@ -171,6 +172,7 @@ def _build_runtime( update_automation_progress=progress_callback, automation_engine=None, missing_download_executor=executor, + album_bundle_executor=album_bundle_executor, run_full_missing_tracks_process=lambda *args, **kwargs: None, get_batch_max_concurrent=lambda: 4, get_active_server=lambda: active_server, @@ -466,3 +468,86 @@ def test_process_wishlist_automatically_skips_when_wishlist_batch_is_already_act assert guard_events == ["enter", "exit"] assert [kwargs.get("progress") for _args, kwargs in progress_calls if "progress" in kwargs] == [10] assert any("already active in another batch" in msg for msg in logger.info_messages) + + +# --- #740: album-bundle batches must route to the dedicated pool ------------ + +def _two_album_tracks_plus_orphan(): + """2 missing tracks from one album (→ album-bundle batch) + 1 orphan + track with no album metadata (→ residual per-track batch).""" + return [ + { + "name": "Album Song 1", + "artists": [{"name": "Artist 1"}], + "spotify_data": { + "album": {"id": "alb1", "name": "Album One", "album_type": "album"}, + "artists": [{"name": "Artist 1"}], + }, + }, + { + "name": "Album Song 2", + "artists": [{"name": "Artist 1"}], + "spotify_data": { + "album": {"id": "alb1", "name": "Album One", "album_type": "album"}, + "artists": [{"name": "Artist 1"}], + }, + }, + { + "name": "Orphan", + "artists": [{"name": "X"}], + "spotify_data": {"album": {"album_type": "album"}, "artists": [{"name": "X"}]}, + }, + ] + + +def test_album_subbatches_route_to_dedicated_album_pool(): + """#740: per-album bundle batches (which block their worker thread for the + whole search+download) must be submitted to the dedicated album_bundle_executor, + NOT the shared missing_download_executor — so a burst of album batches can't + starve the per-track flow / manual wishlist. The residual per-track batch + still goes to the shared pool.""" + album_executor = _FakeExecutor() + batch_map = {} + runtime, _s, _p, _db, shared_executor, _l, _pr, _g = _build_runtime( + tracks=_two_album_tracks_plus_orphan(), + cycle_value="albums", + count=3, + batch_map=batch_map, + album_bundle_executor=album_executor, + ) + + process_wishlist_automatically(runtime, automation_id="auto-pool") + + # Album sub-batch → dedicated album pool; residual → shared pool. + assert len(album_executor.submissions) == 1 + assert len(shared_executor.submissions) == 1 + + album_batch_id = album_executor.submissions[0][1][0] + assert batch_map[album_batch_id]["is_album_download"] is True + assert batch_map[album_batch_id]["album_context"]["name"] == "Album One" + + residual_batch_id = shared_executor.submissions[0][1][0] + assert batch_map[residual_batch_id].get("is_album_download") is not True + + +def test_album_subbatches_fall_back_to_shared_pool_when_no_album_pool(): + """Bulletproofing: if no dedicated album pool is wired (older callers / + tests), album batches fall back to the shared executor — i.e. exactly the + pre-fix behavior, so nothing breaks.""" + batch_map = {} + runtime, _s, _p, _db, shared_executor, _l, _pr, _g = _build_runtime( + tracks=_two_album_tracks_plus_orphan(), + cycle_value="albums", + count=3, + batch_map=batch_map, + album_bundle_executor=None, # not wired + ) + + process_wishlist_automatically(runtime, automation_id="auto-fallback") + + # Both the album sub-batch and the residual land on the shared pool. + assert len(shared_executor.submissions) == 2 + assert any( + batch_map[args[0]].get("is_album_download") is True + for _fn, args, _kw in shared_executor.submissions + ) diff --git a/web_server.py b/web_server.py index 7c51d140..c6332ae5 100644 --- a/web_server.py +++ b/web_server.py @@ -845,6 +845,16 @@ retag_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="RetagWork # Shared task/batch state now lives in core.runtime_state. missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker") +# Dedicated pool for per-album bundle downloads (#740). An album-bundle batch +# blocks its worker thread for the entire search+download; if these run on the +# shared missing_download_executor, a burst of album batches (e.g. a large +# Album-Completeness "Fix all" → wishlist that splits into ~one batch per album) +# saturates all 3 workers and starves the per-track flow AND the user's manual +# "Download Wishlist" analysis, which then never starts. Keeping them on their +# own bounded pool decouples that: hung/slow album downloads can only delay +# other album downloads, never the user-facing path. +album_bundle_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="AlbumBundleWorker") + # Parallelizes the per-file metadata-lookup + post-processing in # /api/import/singles/process. Single-file work is dominated by # Spotify/iTunes/Deezer search round-trips so 3 workers give a near- @@ -1663,6 +1673,7 @@ def _shutdown_runtime_components(): (retag_executor, "retag executor"), (sync_executor, "sync executor"), (missing_download_executor, "missing download executor"), + (album_bundle_executor, "album bundle executor"), (import_singles_executor, "import singles executor"), (tidal_discovery_executor, "tidal discovery executor"), (deezer_discovery_executor, "deezer discovery executor"), @@ -14290,6 +14301,7 @@ def _process_wishlist_automatically(automation_id=None): update_automation_progress=_update_automation_progress, automation_engine=automation_engine, missing_download_executor=missing_download_executor, + album_bundle_executor=album_bundle_executor, run_full_missing_tracks_process=_run_full_missing_tracks_process, get_batch_max_concurrent=_get_batch_max_concurrent, get_active_server=config_manager.get_active_media_server, From e4b5cbbe6040a50f9198a249f4f76fad4c16efc3 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 16:55:31 -0700 Subject: [PATCH 44/52] Wishlist: unify batch-row construction into make_wishlist_batch_row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auto and manual wishlist flows each built the same ~20-field download_batches row in separate places (auto album, auto residual, manual placeholder, manual sub-batches) — four near-identical literals that could (and did) drift apart, producing subtly different batch shapes between the flows. Extract make_wishlist_batch_row() as the single source of truth: it emits the consistent core field set, with the genuinely per-flow differences as explicit arguments — initial phase ('queued' for auto / 'analysis' for manual), the auto-only auto_initiated/auto_processing_timestamp/current_cycle via extra_fields, and album-vs-residual contexts. All four sites now go through it, so every wishlist batch has an IDENTICAL shape (this also removes the field drift that confused the modal-hydration code). Deliberately NOT unified — and left explicit in each caller, per the 'don't cargo-cult genuinely-different code' principle: the grouping decision (auto groups only on the albums cycle), batch-id allocation (manual reuses the caller's placeholder id for the first sub-batch), and dispatch (auto parallel-submits album batches to the dedicated pool + residual to the shared pool; manual runs them serially on one thread). Those are real behavioral differences, not duplication. Behavior-preserving: verified safe to normalize the row shape (grep confirmed every reader uses .get() with defaults, no key-presence checks). The existing auto (test_automation.py) and manual (test_manual_download.py) characterization suites stay green = differential proof of identical behavior. Adds test_batch_factory.py (core fields, album/residual, extra_fields, no shared mutable state, consistent key shape). 131 wishlist tests pass. --- core/wishlist/processing.py | 213 ++++++++++++++------------- tests/wishlist/test_batch_factory.py | 102 +++++++++++++ 2 files changed, 216 insertions(+), 99 deletions(-) create mode 100644 tests/wishlist/test_batch_factory.py diff --git a/core/wishlist/processing.py b/core/wishlist/processing.py index 487401c8..5c50ac62 100644 --- a/core/wishlist/processing.py +++ b/core/wishlist/processing.py @@ -100,6 +100,58 @@ def remove_completed_tracks_from_wishlist( return removed_count +def make_wishlist_batch_row( + *, + playlist_id: str, + playlist_name: str, + track_count: int, + max_concurrent: int, + profile_id: int, + phase: str, + run_id: str | None = None, + is_album: bool = False, + album_context: Optional[Dict[str, Any]] = None, + artist_context: Optional[Dict[str, Any]] = None, + extra_fields: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Single source of truth for a wishlist ``download_batches`` row. + + The auto and manual wishlist flows used to build this ~20-field dict in four + separate places, which let their batch shapes silently drift apart. They now + all go through here so every wishlist batch has an IDENTICAL field shape; the + genuinely per-flow differences (initial ``phase``, the auto-only + ``auto_initiated`` / ``current_cycle`` fields, album vs residual contexts) are + explicit arguments / ``extra_fields``. + + NOTE: this builds the row only — it does NOT decide grouping, batch-id + allocation, or dispatch (parallel-submit vs serial), which legitimately + differ between the flows and stay in their callers. + """ + row: Dict[str, Any] = { + 'phase': phase, + 'playlist_id': playlist_id, + 'playlist_name': playlist_name, + 'queue': [], + 'active_count': 0, + 'max_concurrent': max_concurrent, + 'queue_index': 0, + 'analysis_total': track_count, + 'analysis_processed': 0, + 'analysis_results': [], + 'permanently_failed_tracks': [], + 'cancelled_tracks': set(), + 'force_download_all': True, + 'profile_id': profile_id, + 'is_album_download': is_album, + 'album_context': album_context, + 'artist_context': artist_context, + 'wishlist_run_id': run_id, + } + if extra_fields: + row.update(extra_fields) + return row + + def add_cancelled_tracks_to_failed_tracks( batch: Dict[str, Any], download_tasks: Dict[str, Dict[str, Any]], @@ -455,24 +507,16 @@ def start_manual_wishlist_download_batch( playlist_name = "Wishlist" with runtime.tasks_lock: - runtime.download_batches[batch_id] = { - 'phase': 'analysis', - 'playlist_id': playlist_id, - 'playlist_name': playlist_name, - 'queue': [], - 'active_count': 0, - 'max_concurrent': runtime.get_batch_max_concurrent(), - 'queue_index': 0, - # analysis_total starts at 0; the bg job updates it after cleanup - # finishes and the real track count is known. - 'analysis_total': 0, - 'analysis_processed': 0, - 'analysis_results': [], - 'permanently_failed_tracks': [], - 'cancelled_tracks': set(), - 'force_download_all': True, - 'profile_id': runtime.profile_id, - } + # analysis_total starts at 0; the bg job updates it after cleanup + # finishes and the real track count is known. + runtime.download_batches[batch_id] = make_wishlist_batch_row( + playlist_id=playlist_id, + playlist_name=playlist_name, + track_count=0, + max_concurrent=runtime.get_batch_max_concurrent(), + profile_id=runtime.profile_id, + phase='analysis', + ) runtime.missing_download_executor.submit( _prepare_and_run_manual_wishlist_batch, @@ -631,26 +675,18 @@ def _prepare_and_run_manual_wishlist_batch( runtime.download_batches[batch_id]['artist_context'] = first['artist_context'] runtime.download_batches[batch_id]['playlist_name'] = first['display_name'] for payload in payloads[1:]: - runtime.download_batches[payload['batch_id']] = { - 'phase': 'analysis', - 'playlist_id': 'wishlist', - 'playlist_name': payload['display_name'], - 'queue': [], - 'active_count': 0, - 'max_concurrent': runtime.get_batch_max_concurrent(), - 'queue_index': 0, - 'analysis_total': len(payload['tracks']), - 'analysis_processed': 0, - 'analysis_results': [], - 'permanently_failed_tracks': [], - 'cancelled_tracks': set(), - 'force_download_all': True, - 'profile_id': runtime.profile_id, - 'is_album_download': bool(payload['is_album']), - 'album_context': payload['album_context'], - 'artist_context': payload['artist_context'], - 'wishlist_run_id': wishlist_run_id, - } + runtime.download_batches[payload['batch_id']] = make_wishlist_batch_row( + playlist_id='wishlist', + playlist_name=payload['display_name'], + track_count=len(payload['tracks']), + max_concurrent=runtime.get_batch_max_concurrent(), + profile_id=runtime.profile_id, + phase='analysis', + run_id=wishlist_run_id, + is_album=bool(payload['is_album']), + album_context=payload['album_context'], + artist_context=payload['artist_context'], + ) logger.info( f"[Manual-Wishlist] Split into {len(payloads)} sub-batch(es) " @@ -864,45 +900,29 @@ def process_wishlist_automatically(runtime: WishlistAutoProcessingRuntime, autom f"Wishlist (Auto - Album: {group.album_context.get('name', 'Unknown')})" ) with runtime.tasks_lock: - runtime.download_batches[album_batch_id] = { - # ``queued`` until the master worker - # picks the batch up from the - # ``missing_download_executor`` pool - # (max_workers=3 by default). The worker - # flips phase to ``analysis`` as its - # first action — see - # ``core/downloads/master.py:328``. - # Pre-fix the row was created with - # ``analysis`` directly, so a wishlist - # run with N > 3 sub-batches looked like - # all N were working when really only - # 3 were running. - 'phase': 'queued', - 'playlist_id': playlist_id, - 'playlist_name': album_batch_name, - 'queue': [], - 'active_count': 0, - 'max_concurrent': runtime.get_batch_max_concurrent(), - 'queue_index': 0, - 'analysis_total': len(group.tracks), - 'analysis_processed': 0, - 'analysis_results': [], - 'permanently_failed_tracks': [], - 'cancelled_tracks': set(), - 'force_download_all': True, - 'auto_initiated': True, - 'auto_processing_timestamp': runtime.current_time_fn(), - 'current_cycle': current_cycle, - 'profile_id': runtime.profile_id, - # Album-bundle dispatch gate reads these - # three. With them set, the master worker - # routes through slskd / torrent / usenet - # album-bundle search instead of per-track. - 'is_album_download': True, - 'album_context': group.album_context, - 'artist_context': group.artist_context, - 'wishlist_run_id': wishlist_run_id, - } + # ``queued`` (not ``analysis``) so a run with N > pool + # sub-batches doesn't render all N as "analyzing" while + # only the pool's worth actually run; the master worker + # flips it to ``analysis`` when it picks the batch up. + # is_album_download + contexts route it through the + # album-bundle search instead of per-track. + runtime.download_batches[album_batch_id] = make_wishlist_batch_row( + playlist_id=playlist_id, + playlist_name=album_batch_name, + track_count=len(group.tracks), + max_concurrent=runtime.get_batch_max_concurrent(), + profile_id=runtime.profile_id, + phase='queued', + run_id=wishlist_run_id, + is_album=True, + album_context=group.album_context, + artist_context=group.artist_context, + extra_fields={ + 'auto_initiated': True, + 'auto_processing_timestamp': runtime.current_time_fn(), + 'current_cycle': current_cycle, + }, + ) logger.info( f"[Auto-Wishlist] Album sub-batch {album_idx + 1}/{len(grouping.album_groups)}: " f"'{group.album_context.get('name')}' by '{group.artist_context.get('name')}' " @@ -931,28 +951,23 @@ def process_wishlist_automatically(runtime: WishlistAutoProcessingRuntime, autom batch_id = str(uuid.uuid4()) playlist_name = f"Wishlist (Auto - {current_cycle.capitalize()})" with runtime.tasks_lock: - runtime.download_batches[batch_id] = { - # See album sub-batch above — ``queued`` - # until the master worker picks it up. - 'phase': 'queued', - 'playlist_id': playlist_id, - 'playlist_name': playlist_name, - 'queue': [], - 'active_count': 0, - 'max_concurrent': runtime.get_batch_max_concurrent(), - 'queue_index': 0, - 'analysis_total': len(residual_tracks), - 'analysis_processed': 0, - 'analysis_results': [], - 'permanently_failed_tracks': [], - 'cancelled_tracks': set(), - 'force_download_all': True, - 'auto_initiated': True, - 'auto_processing_timestamp': runtime.current_time_fn(), - 'current_cycle': current_cycle, - 'profile_id': runtime.profile_id, - 'wishlist_run_id': wishlist_run_id, - } + # See album sub-batch above — ``queued`` until the master + # worker picks it up. Residual = classic per-track flow + # (is_album_download defaults False). + runtime.download_batches[batch_id] = make_wishlist_batch_row( + playlist_id=playlist_id, + playlist_name=playlist_name, + track_count=len(residual_tracks), + max_concurrent=runtime.get_batch_max_concurrent(), + profile_id=runtime.profile_id, + phase='queued', + run_id=wishlist_run_id, + extra_fields={ + 'auto_initiated': True, + 'auto_processing_timestamp': runtime.current_time_fn(), + 'current_cycle': current_cycle, + }, + ) _submitted_batches.append(batch_id) runtime.missing_download_executor.submit( runtime.run_full_missing_tracks_process, diff --git a/tests/wishlist/test_batch_factory.py b/tests/wishlist/test_batch_factory.py new file mode 100644 index 00000000..034ee50f --- /dev/null +++ b/tests/wishlist/test_batch_factory.py @@ -0,0 +1,102 @@ +"""Tests for make_wishlist_batch_row — the single source of truth for a wishlist +download_batches row, shared by the auto and manual flows so their batch shapes +can't drift apart. +""" + +from __future__ import annotations + +from core.wishlist.processing import make_wishlist_batch_row + + +_CORE_KEYS = { + 'phase', 'playlist_id', 'playlist_name', 'queue', 'active_count', + 'max_concurrent', 'queue_index', 'analysis_total', 'analysis_processed', + 'analysis_results', 'permanently_failed_tracks', 'cancelled_tracks', + 'force_download_all', 'profile_id', 'is_album_download', 'album_context', + 'artist_context', 'wishlist_run_id', +} + + +def _row(**overrides): + base = dict( + playlist_id='wishlist', playlist_name='Wishlist', track_count=3, + max_concurrent=4, profile_id=1, phase='analysis', + ) + base.update(overrides) + return make_wishlist_batch_row(**base) + + +def test_core_fields_always_present_and_consistent(): + row = _row() + assert _CORE_KEYS <= set(row.keys()) + # Fresh-batch invariants. + assert row['queue'] == [] and row['active_count'] == 0 and row['queue_index'] == 0 + assert row['analysis_processed'] == 0 + assert row['analysis_results'] == [] and row['permanently_failed_tracks'] == [] + assert row['cancelled_tracks'] == set() + assert row['force_download_all'] is True + assert row['analysis_total'] == 3 + assert row['max_concurrent'] == 4 + assert row['profile_id'] == 1 + + +def test_residual_defaults_are_per_track(): + row = _row() + assert row['is_album_download'] is False + assert row['album_context'] is None and row['artist_context'] is None + assert row['wishlist_run_id'] is None + + +def test_album_batch_carries_context(): + row = _row( + phase='queued', run_id='run-1', is_album=True, + album_context={'name': 'Album One'}, artist_context={'name': 'Artist 1'}, + ) + assert row['phase'] == 'queued' + assert row['is_album_download'] is True + assert row['album_context'] == {'name': 'Album One'} + assert row['artist_context'] == {'name': 'Artist 1'} + assert row['wishlist_run_id'] == 'run-1' + + +def test_extra_fields_merged_for_auto(): + row = _row(extra_fields={ + 'auto_initiated': True, 'auto_processing_timestamp': 123.0, + 'current_cycle': 'albums', + }) + assert row['auto_initiated'] is True + assert row['auto_processing_timestamp'] == 123.0 + assert row['current_cycle'] == 'albums' + + +def test_manual_row_has_no_auto_fields(): + """Manual rows must not carry the auto-only fields (no extra_fields).""" + row = _row(phase='analysis') + assert 'auto_initiated' not in row + assert 'current_cycle' not in row + + +def test_fresh_rows_do_not_share_mutable_state(): + """Each row must get its OWN queue/list/set — not a shared reference that + one batch's tasks could leak into another's.""" + a = _row() + b = _row() + a['queue'].append('task-1') + a['cancelled_tracks'].add('x') + assert b['queue'] == [] + assert b['cancelled_tracks'] == set() + assert b['analysis_results'] == [] + + +def test_auto_and_manual_rows_share_identical_key_shape(): + """The drift-prevention guarantee: an auto album row and a manual album row + expose the same set of keys (modulo the auto-only extras), so the modal / + status code sees a consistent shape from both flows.""" + manual = _row(phase='analysis', run_id='r', is_album=True, + album_context={'name': 'A'}, artist_context={'name': 'B'}) + auto = _row(phase='queued', run_id='r', is_album=True, + album_context={'name': 'A'}, artist_context={'name': 'B'}, + extra_fields={'auto_initiated': True, 'current_cycle': 'albums'}) + # Auto is a strict superset (the auto-only extras); the shared core is identical. + assert set(manual.keys()) <= set(auto.keys()) + assert set(auto.keys()) - set(manual.keys()) == {'auto_initiated', 'current_cycle'} From db1e51109c33a28ce853e542f0aaca539172c145 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 17:22:00 -0700 Subject: [PATCH 45/52] Wishlist: extract shared _run_wishlist_cycle engine; auto delegates to it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 1 of unifying the auto + manual wishlist flows. Extract the group -> per-album+residual batches -> register -> dispatch logic that lived inline in process_wishlist_automatically into a standalone _run_wishlist_cycle() engine (built on make_wishlist_batch_row). The auto path now just calls it. Per-flow differences are arguments (auto_initiated stamps the auto-only fields + selects auto vs manual naming/logging; first_batch_id lets a caller reuse a pre-created placeholder). Album batches dispatch to the dedicated album pool, residual to the shared pool (unchanged from #740). Auto behavior is PROVABLY unchanged: its full characterization suite (test_automation.py) stays green (10/10), and the whole wishlist suite passes (131). This commit does NOT touch the manual flow yet (Stage 2) and does not change what auto does — it only moves auto's logic behind a shared entrypoint the manual flow will call next. --- core/wishlist/processing.py | 269 +++++++++++++++++++++--------------- 1 file changed, 158 insertions(+), 111 deletions(-) diff --git a/core/wishlist/processing.py b/core/wishlist/processing.py index 5c50ac62..24e252f9 100644 --- a/core/wishlist/processing.py +++ b/core/wishlist/processing.py @@ -152,6 +152,150 @@ def make_wishlist_batch_row( return row +def _run_wishlist_cycle( + runtime, + *, + playlist_id: str, + cycle: str, + tracks: list, + run_id: str, + auto_initiated: bool, + first_batch_id: Optional[str] = None, +) -> Dict[str, Any]: + """THE single wishlist orchestration engine — both the auto timer and the + manual trigger call this, so a manual scan runs the exact same code path as + an auto scan (group → per-album + residual batches → register → dispatch). + + Per-flow differences are arguments, not separate code: + * ``auto_initiated`` stamps the auto-only fields (auto_initiated / + auto_processing_timestamp / current_cycle, which also drives the + once-per-run cycle toggle on completion) and selects the auto vs manual + display-name + log style. + * ``first_batch_id`` lets the manual flow reuse its synchronously-created + placeholder batch so the modal's existing poll target stays valid. + + Album batches block their worker for the whole search+download, so they run + on the dedicated album pool; the residual per-track batch runs on the shared + pool. Returns a summary dict (submitted ids + album / residual counts). + """ + logger = runtime.logger + from core.wishlist.album_grouping import group_wishlist_tracks_by_album + + # Albums cycle splits into per-album bundles; singles keep the single + # per-track batch shape (Spotify already classifies them away from albums). + grouping = ( + group_wishlist_tracks_by_album( + tracks, min_tracks_per_album=_resolve_album_bundle_threshold(), + ) + if cycle == 'albums' else None + ) + + extra_fields = None + if auto_initiated: + extra_fields = { + 'auto_initiated': True, + 'auto_processing_timestamp': runtime.current_time_fn(), + 'current_cycle': cycle, + } + + # Reuse the caller-provided placeholder id for the FIRST batch created; every + # other batch gets a fresh uuid. + _reuse_id = first_batch_id + + def _alloc_id() -> str: + nonlocal _reuse_id + if _reuse_id is not None: + bid, _reuse_id = _reuse_id, None + return bid + return str(uuid.uuid4()) + + album_executor = runtime.album_bundle_executor or runtime.missing_download_executor + submitted: list = [] + + album_groups = grouping.album_groups if grouping else [] + for album_idx, group in enumerate(album_groups): + album_batch_id = _alloc_id() + album_name = group.album_context.get('name', 'Unknown') + batch_name = ( + f"Wishlist (Auto - Album: {album_name})" if auto_initiated + else f"Wishlist (Album: {album_name})" + ) + with runtime.tasks_lock: + runtime.download_batches[album_batch_id] = make_wishlist_batch_row( + playlist_id=playlist_id, + playlist_name=batch_name, + track_count=len(group.tracks), + max_concurrent=runtime.get_batch_max_concurrent(), + profile_id=runtime.profile_id, + phase='queued', + run_id=run_id, + is_album=True, + album_context=group.album_context, + artist_context=group.artist_context, + extra_fields=extra_fields, + ) + if auto_initiated: + logger.info( + f"[Auto-Wishlist] Album sub-batch {album_idx + 1}/{len(album_groups)}: " + f"'{album_name}' by '{group.artist_context.get('name')}' " + f"({len(group.tracks)} tracks) → {album_batch_id} [run {run_id[:8]}]" + ) + else: + logger.info( + f"[Manual-Wishlist] Album sub-batch {album_idx + 1}/{len(album_groups)}: " + f"'{album_name}' ({len(group.tracks)} tracks) → {album_batch_id}" + ) + submitted.append(album_batch_id) + # Album bundles block their worker for the whole search+download → dedicated + # pool (falls back to the shared pool when unset). See #740. + album_executor.submit( + runtime.run_full_missing_tracks_process, + album_batch_id, playlist_id, group.tracks, + ) + + residual_tracks = grouping.residual_tracks if grouping is not None else tracks + residual_count = len(residual_tracks) if residual_tracks else 0 + if residual_tracks: + residual_batch_id = _alloc_id() + residual_name = ( + f"Wishlist (Auto - {cycle.capitalize()})" if auto_initiated + else "Wishlist (Residual)" + ) + with runtime.tasks_lock: + runtime.download_batches[residual_batch_id] = make_wishlist_batch_row( + playlist_id=playlist_id, + playlist_name=residual_name, + track_count=residual_count, + max_concurrent=runtime.get_batch_max_concurrent(), + profile_id=runtime.profile_id, + phase='queued', + run_id=run_id, + extra_fields=extra_fields, + ) + submitted.append(residual_batch_id) + runtime.missing_download_executor.submit( + runtime.run_full_missing_tracks_process, + residual_batch_id, playlist_id, residual_tracks, + ) + if auto_initiated: + logger.info( + f"Starting wishlist residual batch {residual_batch_id} with {residual_count} tracks " + f"({'singles' if cycle == 'singles' else 'unbucketed albums'}) " + f"[run {run_id[:8]}]" + ) + else: + logger.info( + f"[Manual-Wishlist] Residual per-track batch {residual_batch_id} " + f"with {residual_count} tracks" + ) + + return { + 'submitted': submitted, + 'album_batches': len(album_groups), + 'residual_count': residual_count, + } + + def add_cancelled_tracks_to_failed_tracks( batch: Dict[str, Any], download_tasks: Dict[str, Dict[str, Any]], @@ -869,121 +1013,24 @@ def process_wishlist_automatically(runtime: WishlistAutoProcessingRuntime, autom for i, track in enumerate(wishlist_tracks): track['_original_index'] = i - # When the cycle is 'albums', try to split the wishlist - # into per-album sub-batches so each album fires ONE - # album-bundle search (slskd / torrent / usenet) instead - # of N per-track searches. Residual tracks (no resolvable - # album metadata) fall through to a normal per-track - # batch. Singles cycle keeps its original single-batch - # shape — Spotify already classifies them away from - # albums. - _submitted_batches: list[str] = [] - if current_cycle == 'albums': - from core.wishlist.album_grouping import group_wishlist_tracks_by_album - grouping = group_wishlist_tracks_by_album( - wishlist_tracks, - min_tracks_per_album=_resolve_album_bundle_threshold(), - ) - else: - grouping = None - - # Reify "wishlist run" — one shared id stamped on every - # sub-batch this invocation produces. The completion - # handler uses it to gate the once-per-run cycle toggle - # (so it doesn't fire N times for N sub-batches). + # Reify one "wishlist run" id (the completion handler gates the + # once-per-run cycle toggle on it) and hand off to the SHARED + # wishlist engine — the same code path the manual trigger uses. wishlist_run_id = str(uuid.uuid4()) - - if grouping and grouping.album_groups: - for album_idx, group in enumerate(grouping.album_groups): - album_batch_id = str(uuid.uuid4()) - album_batch_name = ( - f"Wishlist (Auto - Album: {group.album_context.get('name', 'Unknown')})" - ) - with runtime.tasks_lock: - # ``queued`` (not ``analysis``) so a run with N > pool - # sub-batches doesn't render all N as "analyzing" while - # only the pool's worth actually run; the master worker - # flips it to ``analysis`` when it picks the batch up. - # is_album_download + contexts route it through the - # album-bundle search instead of per-track. - runtime.download_batches[album_batch_id] = make_wishlist_batch_row( - playlist_id=playlist_id, - playlist_name=album_batch_name, - track_count=len(group.tracks), - max_concurrent=runtime.get_batch_max_concurrent(), - profile_id=runtime.profile_id, - phase='queued', - run_id=wishlist_run_id, - is_album=True, - album_context=group.album_context, - artist_context=group.artist_context, - extra_fields={ - 'auto_initiated': True, - 'auto_processing_timestamp': runtime.current_time_fn(), - 'current_cycle': current_cycle, - }, - ) - logger.info( - f"[Auto-Wishlist] Album sub-batch {album_idx + 1}/{len(grouping.album_groups)}: " - f"'{group.album_context.get('name')}' by '{group.artist_context.get('name')}' " - f"({len(group.tracks)} tracks) → {album_batch_id} [run {wishlist_run_id[:8]}]" - ) - _submitted_batches.append(album_batch_id) - # Album bundles block their worker thread for the whole - # search+download, so run them on the dedicated album - # pool — never the shared pool that serves analysis, - # per-track downloads and the manual wishlist (#740). - # Fall back to the shared pool if unset (older callers). - _album_executor = ( - runtime.album_bundle_executor or runtime.missing_download_executor - ) - _album_executor.submit( - runtime.run_full_missing_tracks_process, - album_batch_id, playlist_id, group.tracks, - ) - - # Residual tracks (no album group could be formed, OR - # singles cycle): one classic per-track batch as before. - residual_tracks = ( - grouping.residual_tracks if grouping is not None else wishlist_tracks + _cycle_result = _run_wishlist_cycle( + runtime, + playlist_id=playlist_id, + cycle=current_cycle, + tracks=wishlist_tracks, + run_id=wishlist_run_id, + auto_initiated=True, ) - if residual_tracks: - batch_id = str(uuid.uuid4()) - playlist_name = f"Wishlist (Auto - {current_cycle.capitalize()})" - with runtime.tasks_lock: - # See album sub-batch above — ``queued`` until the master - # worker picks it up. Residual = classic per-track flow - # (is_album_download defaults False). - runtime.download_batches[batch_id] = make_wishlist_batch_row( - playlist_id=playlist_id, - playlist_name=playlist_name, - track_count=len(residual_tracks), - max_concurrent=runtime.get_batch_max_concurrent(), - profile_id=runtime.profile_id, - phase='queued', - run_id=wishlist_run_id, - extra_fields={ - 'auto_initiated': True, - 'auto_processing_timestamp': runtime.current_time_fn(), - 'current_cycle': current_cycle, - }, - ) - _submitted_batches.append(batch_id) - runtime.missing_download_executor.submit( - runtime.run_full_missing_tracks_process, - batch_id, playlist_id, residual_tracks, - ) - logger.info( - f"Starting wishlist residual batch {batch_id} with {len(residual_tracks)} tracks " - f"({'singles' if current_cycle == 'singles' else 'unbucketed albums'}) " - f"[run {wishlist_run_id[:8]}]" - ) _summary_parts: list[str] = [] - if grouping and grouping.album_groups: - _summary_parts.append(f"{len(grouping.album_groups)} album batch(es)") - if residual_tracks: - _summary_parts.append(f"{len(residual_tracks)} per-track") + if _cycle_result['album_batches']: + _summary_parts.append(f"{_cycle_result['album_batches']} album batch(es)") + if _cycle_result['residual_count']: + _summary_parts.append(f"{_cycle_result['residual_count']} per-track") _summary_text = ', '.join(_summary_parts) or 'no batches' runtime.update_automation_progress( automation_id, progress=50, From d3c897fb9d2efceed8a45c5c895fbfe29c0fcb08 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 17:43:40 -0700 Subject: [PATCH 46/52] Wishlist: route the manual flow through the shared engine (manual == auto) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 2: the manual 'Download Wishlist' flow now calls the same _run_wishlist_cycle engine the auto timer uses, so a manual scan runs the exact same code path as an auto scan. The old bespoke manual orchestration (build payloads + SERIAL inline dispatch) is deleted — its grouping/dispatch was a near-duplicate of auto's that had already drifted. Behavior changes (all intended, discussed): - Manual now dispatches album bundles in PARALLEL (album pool) like auto, instead of serially on one thread. A single cycle='albums' engine call covers the whole selection (albums bundled, singles/ungroupable -> per-track residual), so no 'both cycles' pass is needed. - The manual placeholder batch_id is reused as the engine's first sub-batch (first_batch_id), so the modal's existing poll target stays valid. - WishlistManualDownloadRuntime gains album_bundle_executor (wired in web_server, falls back to the shared pool when unset). - 'Don't start manual while auto is running' is unchanged — the existing route guard (is_wishlist_actually_processing -> 409) already covers it; no queue added. NOT touched: process_wishlist_automatically's behavior (proven by test_automation staying green in Stage 1) and the per-track download mechanics. test_manual_download.py rewritten to characterize the new behavior (engine dispatch via the executor, parallel, placeholder reuse, album-context). Full wishlist suite green (131); wishlist + automation = 392 passed. --- core/wishlist/processing.py | 121 +++++-------------------- tests/wishlist/test_manual_download.py | 56 ++++++++---- web_server.py | 1 + 3 files changed, 64 insertions(+), 114 deletions(-) diff --git a/core/wishlist/processing.py b/core/wishlist/processing.py index 24e252f9..7cfe4a95 100644 --- a/core/wishlist/processing.py +++ b/core/wishlist/processing.py @@ -627,6 +627,9 @@ class WishlistManualDownloadRuntime: active_server: str profile_id: int logger: Any = module_logger + # Dedicated album-bundle pool, shared with the auto flow via + # _run_wishlist_cycle. Falls back to missing_download_executor when unset. + album_bundle_executor: Any = None def start_manual_wishlist_download_batch( @@ -748,110 +751,34 @@ def _prepare_and_run_manual_wishlist_batch( runtime.add_activity_item("", "Wishlist Download Started", f"{len(wishlist_tracks)} tracks", "Now") - # Try to split into per-album sub-batches so each album fires - # ONE slskd / torrent / usenet album-bundle search (gates on - # ``is_album_download`` + populated album/artist context). - # When a single category was requested (or no category filter) - # we apply the same grouping the auto-wishlist path uses. - # Tracks the grouper can't bucket fall through to a residual - # batch with the classic per-track flow. - from core.wishlist.album_grouping import group_wishlist_tracks_by_album - grouping = group_wishlist_tracks_by_album( - wishlist_tracks, - min_tracks_per_album=_resolve_album_bundle_threshold(), - ) - - # Build the final payload list (batch_id, tracks, album_context, - # artist_context, is_album). The first payload re-uses the - # caller-allocated ``batch_id`` so the frontend's existing poll - # against it keeps working. Subsequent payloads get fresh ids. - payloads = [] - for group in grouping.album_groups: - payloads.append({ - 'tracks': group.tracks, - 'is_album': True, - 'album_context': group.album_context, - 'artist_context': group.artist_context, - 'display_name': f"Wishlist (Album: {group.album_context.get('name', 'Unknown')})", - }) - if grouping.residual_tracks: - payloads.append({ - 'tracks': grouping.residual_tracks, - 'is_album': False, - 'album_context': None, - 'artist_context': None, - 'display_name': "Wishlist (Residual)", - }) - - if not payloads: - # Nothing to download — clear out the original batch. + if not wishlist_tracks: + # Nothing to download — clear out the placeholder batch. with runtime.tasks_lock: if batch_id in runtime.download_batches: runtime.download_batches[batch_id]['analysis_total'] = 0 runtime.download_batches[batch_id]['phase'] = 'complete' return - # Attach the original batch_id to the first payload; allocate - # fresh batch_ids for the rest. - payloads[0]['batch_id'] = batch_id - for payload in payloads[1:]: - payload['batch_id'] = str(uuid.uuid4()) - - # Reify "wishlist run" — one shared id stamped on every sub- - # batch this manual invocation produces. Mirrors the auto - # path. Note manual wishlist completion currently doesn't - # toggle the cycle (only auto does), but the id is set anyway - # so future code + UI grouping have a consistent hook. - wishlist_run_id = str(uuid.uuid4()) - - # Materialize each sub-batch's row state up-front so the - # frontend's polling can see them all under the original - # batch's flow. - with runtime.tasks_lock: - if batch_id in runtime.download_batches: - # Re-purpose the existing row for the first payload. - first = payloads[0] - runtime.download_batches[batch_id]['analysis_total'] = len(first['tracks']) - runtime.download_batches[batch_id]['wishlist_run_id'] = wishlist_run_id - if first['is_album']: - runtime.download_batches[batch_id]['is_album_download'] = True - runtime.download_batches[batch_id]['album_context'] = first['album_context'] - runtime.download_batches[batch_id]['artist_context'] = first['artist_context'] - runtime.download_batches[batch_id]['playlist_name'] = first['display_name'] - for payload in payloads[1:]: - runtime.download_batches[payload['batch_id']] = make_wishlist_batch_row( - playlist_id='wishlist', - playlist_name=payload['display_name'], - track_count=len(payload['tracks']), - max_concurrent=runtime.get_batch_max_concurrent(), - profile_id=runtime.profile_id, - phase='analysis', - run_id=wishlist_run_id, - is_album=bool(payload['is_album']), - album_context=payload['album_context'], - artist_context=payload['artist_context'], - ) - - logger.info( - f"[Manual-Wishlist] Split into {len(payloads)} sub-batch(es) " - f"({sum(1 for p in payloads if p['is_album'])} album + " - f"{sum(1 for p in payloads if not p['is_album'])} residual)" + # Run the selection through the SHARED engine — the exact code path the + # auto timer uses (group → album bundles + per-track residual → parallel + # dispatch on the album / shared pools). cycle='albums' bundles whatever + # forms an album and drops the rest (singles / ungroupable) into the + # per-track residual, so this single call covers the whole selection. + # The placeholder batch_id is reused as the first sub-batch so the + # modal's existing poll target stays valid. + result = _run_wishlist_cycle( + runtime, + playlist_id='wishlist', + cycle='albums', + tracks=wishlist_tracks, + run_id=str(uuid.uuid4()), + auto_initiated=False, + first_batch_id=batch_id, + ) + logger.info( + f"[Manual-Wishlist] Dispatched {result['album_batches']} album batch(es) + " + f"{result['residual_count']} residual track(s) via the shared engine" ) - # Serial dispatch — each album-bundle search happens one at a - # time so the slskd / Prowlarr pipeline doesn't fan out across - # multiple parallel release searches. - for payload in payloads: - label = ( - f"album '{payload['album_context'].get('name')}'" - if payload['is_album'] else 'residual per-track' - ) - logger.info( - f"[Manual-Wishlist] Running sub-batch {payload['batch_id']} " - f"({label}, {len(payload['tracks'])} tracks)" - ) - runtime.run_full_missing_tracks_process( - payload['batch_id'], "wishlist", payload['tracks'], - ) except Exception as exc: logger.error(f"Error preparing manual wishlist batch {batch_id}: {exc}") diff --git a/tests/wishlist/test_manual_download.py b/tests/wishlist/test_manual_download.py index 7e1386ad..bb3429c1 100644 --- a/tests/wishlist/test_manual_download.py +++ b/tests/wishlist/test_manual_download.py @@ -110,6 +110,17 @@ def _run_submitted_bg_job(executor): fn(*args, **kwargs) +def _dispatched(executor, runtime): + """The run_full_missing_tracks_process dispatches the shared engine submitted + to the executor (everything after the initial bg-job submission). Manual now + parallel-dispatches via the engine instead of running them serially inline, + so the master worker is *submitted*, not called directly.""" + return [ + s for s in executor.submissions + if s[0] is runtime.run_full_missing_tracks_process + ] + + def test_start_manual_wishlist_download_batch_returns_immediately_with_placeholder(): """Endpoint returns 200 immediately; cleanup runs in the bg job.""" runtime, service, _db, executor, _logger, activity_calls, batch_map, master_calls = _build_runtime( @@ -174,11 +185,15 @@ def test_start_manual_wishlist_download_batch_filters_track_ids_and_starts_batch _run_submitted_bg_job(executor) assert activity_calls == [("", "Wishlist Download Started", "1 tracks", "Now")] - assert len(master_calls) == 1 - master_args, _ = master_calls[0] - assert master_args[1] == "wishlist" - assert master_args[2][0]["id"] == "track-2" - assert master_args[2][0]["_original_index"] == 0 + # One track → no album group (threshold 2) → one residual batch, dispatched + # via the shared engine (submitted to the executor, not called inline). + dispatched = _dispatched(executor, runtime) + assert len(dispatched) == 1 + args = dispatched[0][1] + assert args[1] == "wishlist" + assert args[2][0]["id"] == "track-2" + assert args[2][0]["_original_index"] == 0 + assert args[0] == payload["batch_id"] # placeholder batch_id reused as first sub-batch assert batch_map[payload["batch_id"]]["analysis_total"] == 1 assert batch_map[payload["batch_id"]]["force_download_all"] is True assert any("Filtered to 1 specific tracks by ID" in msg for msg in logger.info_messages) @@ -224,10 +239,12 @@ def test_start_manual_wishlist_download_batch_does_not_run_library_cleanup(): # The library check is skipped entirely — no per-track DB lookups. assert db.track_checks == [] - # All tracks are submitted to the master worker — including the "owned" one. - assert len(master_calls) == 1 - master_args, _ = master_calls[0] - assert [track["id"] for track in master_args[2]] == ["enhance-1", "owned-1"] + # All tracks are dispatched to the master worker — including the "owned" one. + # Two single-track albums → no album groups → one residual batch of both. + dispatched = _dispatched(executor, runtime) + assert len(dispatched) == 1 + args = dispatched[0][1] + assert [track["id"] for track in args[2]] == ["enhance-1", "owned-1"] assert batch_map[payload["batch_id"]]["analysis_total"] == 2 assert activity_calls == [("", "Wishlist Download Started", "2 tracks", "Now")] @@ -293,28 +310,33 @@ def test_manual_wishlist_splits_into_per_album_sub_batches(): assert status == 200 _run_submitted_bg_job(executor) - # Two album groups → two master-worker calls. - assert len(master_calls) == 2 + # Two album groups → two album sub-batches, PARALLEL-dispatched via the shared + # engine (same as auto) — not serial inline calls. + dispatched = _dispatched(executor, runtime) + assert len(dispatched) == 2 + assert master_calls == [] # nothing run synchronously inline anymore - # First sub-batch uses the caller-allocated batch_id. - first_args, _ = master_calls[0] + # First sub-batch reuses the caller-allocated placeholder batch_id. + first_args = dispatched[0][1] assert first_args[0] == payload["batch_id"] assert batch_map[payload["batch_id"]].get("is_album_download") is True + # Both dispatched batches are album bundles. + for _fn, args, _kw in dispatched: + assert batch_map[args[0]].get("is_album_download") is True # Second sub-batch gets a fresh uuid; its row exists in batch_map. - second_args, _ = master_calls[1] + second_args = dispatched[1][1] assert second_args[0] != payload["batch_id"] assert second_args[0] in batch_map - assert batch_map[second_args[0]].get("is_album_download") is True # Track counts across the two sub-batches: 2 each at threshold=2. - counts = sorted(len(args[2]) for args, _ in master_calls) + counts = sorted(len(args[2]) for _fn, args, _kw in dispatched) assert counts == [2, 2] # Both sub-batches carry album context populated from spotify_data. album_names = { batch_map[args[0]]["album_context"]["name"] - for args, _ in master_calls + for _fn, args, _kw in dispatched } assert album_names == {"Album One", "Album Two"} diff --git a/web_server.py b/web_server.py index c6332ae5..0da4c37e 100644 --- a/web_server.py +++ b/web_server.py @@ -15155,6 +15155,7 @@ def start_wishlist_missing_downloads(): download_batches=download_batches, tasks_lock=tasks_lock, missing_download_executor=missing_download_executor, + album_bundle_executor=album_bundle_executor, run_full_missing_tracks_process=_run_full_missing_tracks_process, get_batch_max_concurrent=_get_batch_max_concurrent, add_activity_item=add_activity_item, From cea897cbd14c42ba4f51608df5d971226d20ef69 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 18:31:45 -0700 Subject: [PATCH 47/52] Wishlist: import Optional (fix ruff F821 in processing.py) make_wishlist_batch_row / _run_wishlist_cycle annotate params with Optional, but the typing import only had Any/Callable/Dict. Slipped past py_compile + tests because 'from __future__ import annotations' makes annotations strings (never evaluated at runtime), but ruff flags it statically (F821). --- core/wishlist/processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/wishlist/processing.py b/core/wishlist/processing.py index 7cfe4a95..fb6cc20e 100644 --- a/core/wishlist/processing.py +++ b/core/wishlist/processing.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from datetime import datetime from contextlib import AbstractContextManager from types import SimpleNamespace -from typing import Any, Callable, Dict +from typing import Any, Callable, Dict, Optional from core.wishlist.payloads import build_failed_track_wishlist_context from core.wishlist.selection import filter_wishlist_tracks_by_category, sanitize_and_dedupe_wishlist_tracks From aa2806180e00a09e205f41d088b8f73ea9fe6ccb Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 19:19:28 -0700 Subject: [PATCH 48/52] Fix: Soulseek album poll hangs on a stalled peer; failed batches never cleared MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related bugs from the Slipknot album never finishing. 1) _poll_album_bundle_downloads hung when the peer stalled. The finish check needs every transfer terminal (completed/failed); the #715 grace only covers 'slskd says Completed but file not on disk'. A transfer stuck InProgress / Queued, or dropped by slskd, is none of those — so it blocked both the finish AND the grace exit, and the poll spun to the full ~6h timeout. Add a bundle-level stall guard: track a progress marker (#terminal transfers, total bytes across pending). If NOTHING advances for _stall_grace (180s) — no terminal transition AND no pending byte movement — the peer has stalled; mark the stuck transfers failed so the existing finish/all-failed checks resolve the bundle with whatever completed (missing tracks then fall back to the per-track matcher). Conservative: only trips when EVERYTHING is frozen, so a slow-but-progressing or still-queued transfer is unaffected. 2) Failed batches lingered in the UI forever ('No tracks loaded'). The auto-cleanup gate removed only complete/error/cancelled phases — 'failed' (e.g. an album-bundle hard failure) was missing, so it never aged out. Add 'failed' to the terminal set so it's removed after 5 minutes like the others. Tests (tests/test_soulseek_album_poll_stall.py): stalled peer → gives up with the completed subset (not the deadline); progressing bundle not falsely stalled; all-stalled → empty; dropped transfers stall out; clean finish unaffected. 124 download/soulseek tests pass; ruff clean. --- core/soulseek_client.py | 50 ++++++++ tests/test_soulseek_album_poll_stall.py | 146 ++++++++++++++++++++++++ web_server.py | 6 +- 3 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 tests/test_soulseek_album_poll_stall.py diff --git a/core/soulseek_client.py b/core/soulseek_client.py index 561e4867..e484468f 100644 --- a/core/soulseek_client.py +++ b/core/soulseek_client.py @@ -1700,6 +1700,22 @@ class SoulseekClient(DownloadSourcePlugin): # Seconds an "slskd Completed but locally unresolved" key # has to stay stuck before we give up on it. _unresolved_grace = 45.0 + # Bundle-level stall guard. The #715 grace above only covers + # "slskd says Completed but the file isn't on disk yet". It does NOT + # cover a transfer the peer stalls on — stuck InProgress / Queued, or + # dropped by slskd entirely — which is never failed, never completed, + # and never marked unresolved, so it blocks BOTH the all-terminal + # finish check AND the grace exit, and the poll spun to the full + # ``get_poll_timeout()`` deadline (the Slipknot hang). If NOTHING + # progresses — no transfer completes/fails and no pending transfer's + # byte count moves — for this long, the folder has stalled: mark the + # stuck transfers failed so the bundle resolves with whatever + # completed (the per-track matcher then handles the missing tracks). + # Conservative on purpose: only trips when EVERYTHING is frozen, so a + # slow-but-progressing or still-queued-then-starting transfer is safe. + _stall_grace = 180.0 + _last_progress_marker = None + _last_progress_at = time.monotonic() while time.monotonic() < deadline: try: downloads = run_async(self.get_all_downloads()) @@ -1755,6 +1771,40 @@ class SoulseekClient(DownloadSourcePlugin): count=len(completed_paths), failed=len(failed_states), ) + + # Bundle-level stall detection (see ``_stall_grace`` above). Advance + # the progress marker on ANY forward motion — a transfer reaching a + # terminal state, or a still-pending transfer downloading more bytes. + # If the marker is frozen for ``_stall_grace``, the peer has stalled; + # mark the stuck transfers failed so the finish/all-failed checks + # below resolve the bundle instead of spinning to the deadline. + now = time.monotonic() + stall_pending = [ + k for k in transfer_keys + if k not in completed_paths and k not in failed_states + ] + pending_bytes = 0 + for k in stall_pending: + dl = by_key.get(k) or by_key.get(( + k[0], os.path.basename((k[1] or '').replace('\\', '/')), + )) + pending_bytes += (getattr(dl, 'transferred', 0) or 0) if dl else 0 + marker = (len(completed_paths) + len(failed_states), pending_bytes) + if marker != _last_progress_marker: + _last_progress_marker = marker + _last_progress_at = now + elif stall_pending and (now - _last_progress_at) >= _stall_grace: + logger.warning( + "[Soulseek album] No progress for %.0fs — peer stalled on %d " + "transfer(s) (stuck / queued / dropped). Marking them failed and " + "resolving with what completed; missing tracks fall back to " + "per-track. Stalled: %s", + _stall_grace, len(stall_pending), + [transfer_keys[k].filename for k in stall_pending[:5]], + ) + for k in stall_pending: + failed_states[k] = 'Stalled' + if completed_paths and len(completed_paths) + len(failed_states) == len(transfer_keys): logger.warning( "[Soulseek album] Selected folder finished with %d completed and %d failed transfer(s)", diff --git a/tests/test_soulseek_album_poll_stall.py b/tests/test_soulseek_album_poll_stall.py new file mode 100644 index 00000000..41ea071a --- /dev/null +++ b/tests/test_soulseek_album_poll_stall.py @@ -0,0 +1,146 @@ +"""Regression for the Soulseek album-bundle poll hanging when the peer stalls. + +`_poll_album_bundle_downloads` waits for every transfer in the selected folder +to reach a terminal state (completed or failed). A transfer the peer stalls on — +stuck InProgress/Queued, or dropped by slskd — is never failed, never completed, +and never marked "completed-but-unresolved", so it used to block both the +all-terminal finish check AND the #715 grace exit, and the poll spun to the full +~6h timeout (the Slipknot hang). + +The fix adds a bundle-level stall guard: if NOTHING progresses (no transfer +reaches terminal AND no pending transfer's byte count moves) for `_stall_grace` +seconds, the stuck transfers are marked failed so the bundle resolves with what +completed. These tests drive the real poll with a fake clock + scripted slskd +states. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from core.soulseek_client import SoulseekClient + + +class _Clock: + def __init__(self): + self.now = 0.0 + + def monotonic(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +def _dl(username, filename, state, *, size=100, transferred=100): + return SimpleNamespace( + username=username, filename=filename, state=state, + size=size, transferred=transferred, + ) + + +class _StubClient: + """Minimal stand-in for SoulseekClient with just what the poll touches.""" + + def __init__(self, states, resolvable): + self._states = states # list of download-lists, one per poll; last repeats + self._call = 0 + self._resolvable = set(resolvable) + + def get_all_downloads(self): + i = min(self._call, len(self._states) - 1) + self._call += 1 + return self._states[i] + + def _resolve_downloaded_album_file(self, filename): + base = os.path.basename((filename or "").replace("\\", "/")) + if base in self._resolvable or filename in self._resolvable: + return Path(f"/staged/{base}") + return None + + +def _run_poll(stub, transfer_keys, *, timeout=7200.0, interval=2.0): + clock = _Clock() + emits = [] + with patch("core.soulseek_client.time", clock), \ + patch("core.soulseek_client.run_async", lambda x: x), \ + patch("core.soulseek_client.get_poll_timeout", lambda: timeout), \ + patch("core.soulseek_client.get_poll_interval", lambda: interval): + result = SoulseekClient._poll_album_bundle_downloads( + stub, transfer_keys, lambda phase, **kw: emits.append((phase, kw)) + ) + return result, clock, emits + + +def _keys(*names, user="peer"): + """Build {(user, name): TrackResult-ish} preserving order.""" + return {(user, n): SimpleNamespace(filename=n) for n in names} + + +def test_stalled_peer_gives_up_and_returns_completed_subset(): + tk = _keys("01.flac", "02.flac", "03.flac") + # 01 completes (resolvable); 02/03 stuck InProgress with FROZEN byte counts. + frozen = [ + _dl("peer", "01.flac", "Completed", transferred=100, size=100), + _dl("peer", "02.flac", "InProgress", transferred=50, size=100), + _dl("peer", "03.flac", "InProgress", transferred=30, size=100), + ] + stub = _StubClient([frozen], resolvable={"01.flac"}) + result, clock, _ = _run_poll(stub, tk, timeout=7200.0, interval=2.0) + + # Resolved with the one completed track instead of hanging to the deadline. + assert result == [Path("/staged/01.flac")] + # Gave up around the stall window (~180s), nowhere near the 7200s timeout. + assert clock.now < 600.0 + + +def test_progressing_bundle_is_not_falsely_stalled(): + tk = _keys("01.flac", "02.flac") + # 02 keeps downloading more bytes each poll, then completes — must NOT trip + # the stall guard even though it takes a while. + states = [ + [_dl("peer", "01.flac", "Completed"), _dl("peer", "02.flac", "InProgress", transferred=10, size=100)], + [_dl("peer", "01.flac", "Completed"), _dl("peer", "02.flac", "InProgress", transferred=40, size=100)], + [_dl("peer", "01.flac", "Completed"), _dl("peer", "02.flac", "InProgress", transferred=80, size=100)], + [_dl("peer", "01.flac", "Completed"), _dl("peer", "02.flac", "Completed", transferred=100, size=100)], + ] + stub = _StubClient(states, resolvable={"01.flac", "02.flac"}) + result, _clock, _ = _run_poll(stub, tk, timeout=7200.0, interval=2.0) + + assert set(result) == {Path("/staged/01.flac"), Path("/staged/02.flac")} + + +def test_all_transfers_stalled_returns_empty(): + tk = _keys("01.flac", "02.flac") + frozen = [ + _dl("peer", "01.flac", "InProgress", transferred=10, size=100), + _dl("peer", "02.flac", "Queued", transferred=0, size=100), + ] + stub = _StubClient([frozen], resolvable=set()) + result, clock, _ = _run_poll(stub, tk, timeout=7200.0, interval=2.0) + + assert result == [] # nothing completed → empty (caller falls back) + assert clock.now < 600.0 # didn't spin to the deadline + + +def test_dropped_transfers_also_stall_out(): + """slskd dropping the transfers entirely (dl=None) must also trip the guard, + not hang — there's no byte progress and nothing terminal.""" + tk = _keys("01.flac", "02.flac") + stub = _StubClient([[]], resolvable=set()) # get_all_downloads returns nothing + result, clock, _ = _run_poll(stub, tk, timeout=7200.0, interval=2.0) + + assert result == [] + assert clock.now < 600.0 + + +def test_clean_finish_unaffected(): + tk = _keys("01.flac", "02.flac") + done = [_dl("peer", "01.flac", "Completed"), _dl("peer", "02.flac", "Succeeded")] + stub = _StubClient([done], resolvable={"01.flac", "02.flac"}) + result, clock, _ = _run_poll(stub, tk) + assert set(result) == {Path("/staged/01.flac"), Path("/staged/02.flac")} + assert clock.now < 10.0 # resolves on the first couple polls diff --git a/web_server.py b/web_server.py index 0da4c37e..f767e7e6 100644 --- a/web_server.py +++ b/web_server.py @@ -1418,8 +1418,10 @@ def validate_and_heal_batch_states(): queue = batch_data.get('queue', []) phase = batch_data.get('phase', 'unknown') - # AUTO-CLEANUP: Remove completed batches after 5 minutes to prevent stale state - if phase in ['complete', 'error', 'cancelled']: + # AUTO-CLEANUP: Remove terminal batches after 5 minutes to prevent stale state. + # 'failed' (e.g. an album-bundle hard failure) was missing here, so a failed + # batch lingered in the UI forever ("No tracks loaded") and never cleared. + if phase in ['complete', 'error', 'cancelled', 'failed']: # Check if batch has a completion timestamp completion_time = batch_data.get('completion_time') if not completion_time: From a60eae931555018f6203d408026f4bef7d31fc85 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 19:29:55 -0700 Subject: [PATCH 49/52] Soulseek album poll: treat 'Aborted'/'Cancelled' transfers as failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live testing surfaced that slskd reports a peer-side abort as 'Completed, Aborted' at 0 bytes (peer accepts then drops every transfer). That string contains 'Completed', so the poll's completed-branch ran first and misread it as 'completed but file missing' — routing it into the #715 unresolved/download_path grace (gives up after 45s with a misleading 'download_path mismatch' log) instead of recognizing it as a failure. Add 'Aborted' and 'Cancelled' to the failure-token check (which runs before the completed branch), so these resolve immediately and correctly as failed. Test added for the all-aborted folder. --- core/soulseek_client.py | 9 ++++++++- tests/test_soulseek_album_poll_stall.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/core/soulseek_client.py b/core/soulseek_client.py index e484468f..aa624d1e 100644 --- a/core/soulseek_client.py +++ b/core/soulseek_client.py @@ -1740,7 +1740,14 @@ class SoulseekClient(DownloadSourcePlugin): os.path.basename((key[1] or '').replace('\\', '/')), )) state = (getattr(dl, 'state', '') or '') if dl else '' - if any(token in state for token in ('Errored', 'Failed', 'Rejected', 'TimedOut')): + # NOTE: check failure tokens BEFORE the 'Completed' branch — slskd + # reports terminal failures as "Completed, " (e.g. + # "Completed, Aborted" / "Completed, Cancelled" when a peer accepts + # then drops every transfer at 0 bytes). Those contain "Completed", + # so without catching the failure reason first they'd be misread as + # "completed but file missing" (the #715 download_path path). + if any(token in state for token in + ('Errored', 'Failed', 'Rejected', 'TimedOut', 'Aborted', 'Cancelled')): failed_states[key] = state or 'Failed' logger.warning( "[Soulseek album] Transfer failed from selected folder: %s (%s)", diff --git a/tests/test_soulseek_album_poll_stall.py b/tests/test_soulseek_album_poll_stall.py index 41ea071a..5fc501cc 100644 --- a/tests/test_soulseek_album_poll_stall.py +++ b/tests/test_soulseek_album_poll_stall.py @@ -137,6 +137,22 @@ def test_dropped_transfers_also_stall_out(): assert clock.now < 600.0 +def test_completed_aborted_classified_as_failed_not_unresolved(): + """slskd reports a peer-side abort as 'Completed, Aborted' at 0 bytes. Because + that string contains 'Completed', it was misread as 'completed but file + missing' (#715 path). It must be classified as FAILED — so an all-aborted + folder resolves immediately, not after the unresolved/stall grace.""" + tk = _keys("01.flac", "02.flac") + aborted = [ + _dl("peer", "01.flac", "Completed, Aborted", transferred=0, size=3_600_000), + _dl("peer", "02.flac", "Completed, Aborted", transferred=0, size=24_700_000), + ] + stub = _StubClient([aborted], resolvable=set()) + result, clock, _ = _run_poll(stub, tk, timeout=7200.0, interval=2.0) + assert result == [] # all failed → empty (caller falls back) + assert clock.now < 30.0 # resolved on the first poll, no 45s/180s wait + + def test_clean_finish_unaffected(): tk = _keys("01.flac", "02.flac") done = [_dl("peer", "01.flac", "Completed"), _dl("peer", "02.flac", "Succeeded")] From e94523f3e9320bf0b4d741045efdbe4bad0d4083 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Fri, 29 May 2026 19:46:52 -0700 Subject: [PATCH 50/52] Album bundle: fall back to per-track when the chosen folder yields nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the preflight-selected Soulseek folder produces zero usable files — every transfer failed/aborted/stalled (the Slipknot dead-peer case: all tracks 'Completed, Aborted' at 0 bytes) — _poll_album_bundle_downloads returns []. download_album_to_staging used to return that with fallback=False, so try_dispatch marked the whole batch failed and nothing was retried elsewhere until the next wishlist run. Flip that branch to fallback=True so the existing, proven per-track flow takes over and re-searches every missing track across ALL sources/peers. This reuses the per-track multi-source robustness instead of reimplementing candidate-folder retry inside the bundle. Tests: tests/test_soulseek_album_fallback.py drives the preflight-reuse path with a stubbed poll — empty poll -> fallback=True (differential-verified it fails without the fix), healthy poll -> fallback stays False. Downstream routing (fallback=True -> per-track) already covered by test_album_bundle_dispatch.py. --- core/soulseek_client.py | 10 ++- tests/test_soulseek_album_fallback.py | 100 ++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 tests/test_soulseek_album_fallback.py diff --git a/core/soulseek_client.py b/core/soulseek_client.py index aa624d1e..80d7d4bc 100644 --- a/core/soulseek_client.py +++ b/core/soulseek_client.py @@ -1601,7 +1601,15 @@ class SoulseekClient(DownloadSourcePlugin): result['fallback'] = False completed = self._poll_album_bundle_downloads(transfer_keys, _emit) if not completed: - result['error'] = 'Soulseek album download failed or timed out' + # The selected folder yielded ZERO usable files — every transfer + # failed / aborted / stalled (a dead or unwilling peer). Don't hard- + # fail the batch: fall back to the per-track flow, which searches ALL + # sources per track and can pull each from a live peer. We reuse that + # proven multi-source robustness instead of looping candidate folders + # here. (Per-track only fires for a genuinely-missing album anyway.) + result['error'] = ('Soulseek album folder produced no usable files ' + '(peer failed/aborted/stalled) — falling back to per-track') + result['fallback'] = True return result _emit('staging', release=getattr(picked, 'album_title', folder_path) if picked else folder_path) diff --git a/tests/test_soulseek_album_fallback.py b/tests/test_soulseek_album_fallback.py new file mode 100644 index 00000000..5e2cce07 --- /dev/null +++ b/tests/test_soulseek_album_fallback.py @@ -0,0 +1,100 @@ +"""Regression: a Soulseek album folder that yields ZERO usable files must fall +back to the per-track flow rather than hard-failing the whole batch. + +The Slipknot case — the preflight-selected peer aborts/stalls every transfer (all +tracks reported "Completed, Aborted" at 0 bytes) — makes +``_poll_album_bundle_downloads`` return ``[]``. ``download_album_to_staging`` used +to return that empty result with ``fallback=False``, so the dispatcher +(``core/downloads/album_bundle_dispatch.try_dispatch``) hit its ``mark_failed`` +branch and the batch died with nothing tried elsewhere. + +The fix flips that branch to ``fallback=True`` so the existing, proven per-track +flow takes over and re-searches every missing track across ALL sources/peers — +reusing that robustness instead of looping candidate folders inside the bundle. +The downstream "fallback=True -> per-track" routing is covered by +``tests/test_album_bundle_dispatch.py`` +(``test_dispatch_fallback_failure_returns_false_for_per_track_flow``); these tests +prove the empty-folder branch actually sets the flag, and that a healthy folder +does NOT fall back. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import patch + +from core.soulseek_client import SoulseekClient + + +class _Stub: + """Stand-in exposing only what download_album_to_staging touches on the + preflight-reuse path (preferred_source + preferred_tracks skip search/browse).""" + + def __init__(self, poll_result): + self._poll_result = poll_result + + def is_configured(self): + return True + + def filter_results_by_quality_preference(self, tracks): + return tracks + + def download(self, username, filename, size): + return f"dl-{filename}" # truthy id; run_async patched to identity + + def _poll_album_bundle_downloads(self, transfer_keys, emit): + return self._poll_result + + +def _track(name): + return SimpleNamespace(username="deadpeer", filename=name, size=100) + + +def _run(poll_result): + stub = _Stub(poll_result) + tracks = [_track("01.flac"), _track("02.flac")] + with patch("core.soulseek_client.run_async", lambda x: x): + result = SoulseekClient.download_album_to_staging( + stub, + album_name="All Hope Is Gone", + artist_name="Slipknot", + staging_dir="/tmp/staging-does-not-matter", + preferred_source={ + "username": "deadpeer", + "folder_path": "music/Slipknot/All Hope Is Gone", + }, + preferred_tracks=tracks, + ) + return result, tracks + + +def test_empty_folder_falls_back_to_per_track(): + # Poll returns [] — every transfer aborted / stalled (dead peer). + result, _ = _run([]) + assert result['success'] is False + assert result['fallback'] is True # <-- the fix: hand off to per-track + assert result['files'] == [] + assert 'per-track' in (result['error'] or '') + + +def test_healthy_folder_does_not_fall_back(): + # Positive control: the poll yields staged files and the copy succeeds, so we + # must NOT flip fallback — this proves the empty-branch isn't blanket-returning + # True. Patch the atomic copy to echo the completed paths through. + from pathlib import Path + completed = [Path("/staged/01.flac"), Path("/staged/02.flac")] + with patch("core.soulseek_client.copy_audio_files_atomically", lambda files, dest: list(files)): + stub = _Stub(completed) + with patch("core.soulseek_client.run_async", lambda x: x): + result = SoulseekClient.download_album_to_staging( + stub, + album_name="All Hope Is Gone", + artist_name="Slipknot", + staging_dir="/tmp/staging-does-not-matter", + preferred_source={"username": "goodpeer", "folder_path": "music/Slipknot/AHIG"}, + preferred_tracks=[_track("01.flac"), _track("02.flac")], + ) + assert result['success'] is True + assert result['fallback'] is False + assert result['partial'] is False + assert result['files'] == completed From 20cd12e66b7a580de11268e48d9a664cd4c8f84a Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Sat, 30 May 2026 00:15:06 -0700 Subject: [PATCH 51/52] Reorganize: skip files in the duplicate-cleaner /deleted quarantine (#746) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Duplicate Cleaner moves de-duplicated files into /deleted/. If a user's media server scans the transfer folder (e.g. a /music root holding both the library and the transfer dir), those quarantined files get real track rows in SoulSync's DB. Reorganize is purely DB-driven — it acts on each track's stored file_path — so it would dutifully move a quarantined file back OUT of /deleted to the template location, exactly what Tacobell444 reported. We can't stop the rows from existing (they come from the media server, which the app doesn't control), so the fix is bounded to Reorganize, as the reporter asked: skip any track whose resolved path is under /deleted. Surfaced as a non-matched 'In deleted/quarantine folder — skipped' in the preview; apply mirrors it (post-process never runs, file left in place, counted as skipped). Detection is anchored to the /deleted PREFIX (not a bare substring) so a real album like 'Deleted Scenes' is kept; falls back to an exact 'deleted' path-segment match when transfer_dir is unavailable (mirrors the cleaner's own 'if deleted in dirs' skip). The one unavoidable ambiguity — an artist folder named exactly 'deleted' at the transfer root — is pinned in a test as intentional. Guard added once where both consumers see it: preview_album_reorganize and the apply worker (_RunContext gains transfer_dir). Tests: tests/test_reorganize_deleted_quarantine.py (8 unit) + test_library_reorganize_orchestrator.py (preview + apply integration, differential-verified they fail without the fix). 128 adjacent reorganize tests still green. --- core/library_reorganize.py | 58 +++++++++++ tests/test_library_reorganize_orchestrator.py | 97 +++++++++++++++++++ tests/test_reorganize_deleted_quarantine.py | 72 ++++++++++++++ 3 files changed, 227 insertions(+) create mode 100644 tests/test_reorganize_deleted_quarantine.py diff --git a/core/library_reorganize.py b/core/library_reorganize.py index 2aea45bb..af795789 100644 --- a/core/library_reorganize.py +++ b/core/library_reorganize.py @@ -956,6 +956,17 @@ def preview_album_reorganize( 'disc_number': None, } + # #746: never reorganize files sitting in the duplicate-cleaner + # quarantine (/deleted). Surface as a non-matched skip so + # the preview shows WHY and apply leaves them put. Checked before the + # matched branch so a quarantined track that happens to match the API + # tracklist is still skipped. + if _is_in_deleted_quarantine(resolved, transfer_dir): + item['matched'] = False + item['reason'] = 'In deleted/quarantine folder — skipped' + preview_tracks.append(item) + continue + if not plan_item['matched']: preview_tracks.append(item) continue @@ -1011,6 +1022,42 @@ def preview_album_reorganize( } +def _is_in_deleted_quarantine(resolved_path, transfer_dir) -> bool: + """True when ``resolved_path`` lives inside the duplicate-cleaner + quarantine folder (``/deleted/...``). + + The Duplicate Cleaner (``core/library/duplicate_cleaner.py``) moves + de-duplicated files into ``/deleted/``. If the user's + media server scans the transfer folder (e.g. a ``/music`` root that + contains both the library and the transfer dir), those quarantined + files get real rows in SoulSync's DB — and Reorganize, being purely + DB-driven, would otherwise dutifully move them back OUT of /deleted + to the template location. This guard makes Reorganize skip them so + the quarantine stays quarantined (#746). + + Anchored to ``/deleted`` specifically so a legitimately + named artist/album like "Deleted" elsewhere in the library is NOT + skipped. When ``transfer_dir`` is unavailable we fall back to an exact + ``deleted`` path-SEGMENT match (mirrors the cleaner's own + ``if 'deleted' in dirs`` skip) — never a substring, so "Undeleted" + or "deleted scenes" stay safe. + """ + if not resolved_path: + return False + + def _norm(p): + # normpath collapses redundant separators / '..'; normcase applies + # the platform's case rule (lowercases on Windows, no-op on posix); + # fold to '/' so the segment/prefix checks are separator-agnostic. + return os.path.normcase(os.path.normpath(p)).replace('\\', '/') + + norm = _norm(resolved_path) + if transfer_dir: + quarantine = _norm(os.path.join(transfer_dir, 'deleted')) + return norm == quarantine or norm.startswith(quarantine + '/') + return 'deleted' in norm.split('/') + + def _trim_to_transfer(db_path, resolved, transfer_dir): """Compose the user-facing 'current path' string — relative to the transfer dir if the file lives there, else the raw DB value.""" @@ -1094,6 +1141,7 @@ class _RunContext: update_track_path_fn: Optional[Callable[[Any, str], None]] = None on_progress: Optional[Callable[[dict], None]] = None stop_check: Optional[Callable[[], bool]] = None + transfer_dir: Optional[str] = None # anchors the #746 /deleted-quarantine skip def emit(self, **updates) -> None: """Fire the progress callback. Caller is responsible for @@ -1271,6 +1319,15 @@ def _process_one_track(ctx: _RunContext, plan_item: dict) -> None: f"File not found on disk — DB path: {db_path or '(empty)'}") return + # #746: leave duplicate-cleaner quarantine files (/deleted) + # where they are. Matches the preview's skip so apply never yanks a file + # back out of /deleted. (Mirrors the preview guard in + # preview_album_reorganize.) + if _is_in_deleted_quarantine(resolved_src, ctx.transfer_dir): + ctx.record_error(track_id, title, + 'In deleted/quarantine folder — skipped') + return + staging_file = _stage_track(ctx, track_id, title, resolved_src) if staging_file is None: return @@ -1470,6 +1527,7 @@ def reorganize_album( update_track_path_fn=update_track_path_fn, on_progress=on_progress, stop_check=stop_check, + transfer_dir=transfer_dir, ) try: diff --git a/tests/test_library_reorganize_orchestrator.py b/tests/test_library_reorganize_orchestrator.py index ef33195b..e4045b9f 100644 --- a/tests/test_library_reorganize_orchestrator.py +++ b/tests/test_library_reorganize_orchestrator.py @@ -1934,3 +1934,100 @@ def test_stop_check_aborts_remaining_tracks(monkeypatch, tmpdirs): # but not ALL 10 — the stop_check cut off the unstarted ones. assert pp_count[0] < 10 assert pp_count[0] >= 2 + + +# --- tests: #746 /deleted-quarantine skip --------------------------------- + +def test_preview_skips_track_in_deleted_quarantine(monkeypatch, tmpdirs): + """A track whose file lives in /deleted (duplicate-cleaner + quarantine) must be surfaced as a non-matched skip in the preview, even + though its title matches the API tracklist — Reorganize must not offer to + move it back out of /deleted (#746).""" + library, _staging, transfer = tmpdirs + db = _FakeDB() + # One normal track in the library, one quarantined track under + # /deleted. Both have titles that match the API list. + quarantine = transfer / 'deleted' / 'Aerosmith' + quarantine.mkdir(parents=True) + deleted_file = quarantine / 'dream.flac' + deleted_file.write_bytes(b'dupe') + _setup_album(db, deezer_id='dz-1', tracks=[ + ('t1', 1, 'Same Old Song And Dance', _make_audio_file(library, 't1.flac')), + ('t2', 2, 'Dream On', str(deleted_file)), + ]) + + monkeypatch.setattr(library_reorganize, 'get_primary_source', lambda: 'deezer') + monkeypatch.setattr(library_reorganize, 'get_source_priority', lambda p: [p]) + monkeypatch.setattr(library_reorganize, 'get_album_for_source', + lambda *a: {'id': 'dz-1', 'name': 'Aerosmith'}) + monkeypatch.setattr( + library_reorganize, 'get_album_tracks_for_source', + lambda *a: {'items': [ + {'id': 'a1', 'name': 'Same Old Song And Dance', 'track_number': 1}, + {'id': 'a2', 'name': 'Dream On', 'track_number': 2}, + ]}, + ) + + result = library_reorganize.preview_album_reorganize( + album_id='alb-1', db=db, transfer_dir=str(transfer), + resolve_file_path_fn=lambda p: p, + build_final_path_fn=_fake_path_builder, + ) + + by_title = {it['title']: it for it in result['tracks']} + # Normal track: matched + gets a destination. + assert by_title['Same Old Song And Dance']['matched'] is True + assert by_title['Same Old Song And Dance']['new_path'] + # Quarantined track: skipped despite matching the API tracklist. + assert by_title['Dream On']['matched'] is False + assert 'quarantine' in (by_title['Dream On']['reason'] or '').lower() + assert by_title['Dream On']['new_path'] == '' + + +def test_apply_skips_track_in_deleted_quarantine(monkeypatch, tmpdirs): + """Apply mirrors the preview: post-process is never called for a + quarantined track, the original is left in /deleted, and it's counted as + skipped (not moved, not failed) (#746).""" + library, staging, transfer = tmpdirs + db = _FakeDB() + quarantine = transfer / 'deleted' / 'Aerosmith' + quarantine.mkdir(parents=True) + deleted_file = quarantine / 'dream.flac' + deleted_file.write_bytes(b'dupe') + _setup_album(db, deezer_id='dz-1', tracks=[ + ('t1', 1, 'Same Old Song And Dance', _make_audio_file(library, 't1.flac')), + ('t2', 2, 'Dream On', str(deleted_file)), + ]) + + monkeypatch.setattr(library_reorganize, 'get_primary_source', lambda: 'deezer') + monkeypatch.setattr(library_reorganize, 'get_source_priority', lambda p: [p]) + monkeypatch.setattr(library_reorganize, 'get_album_for_source', + lambda *a: {'id': 'dz-1', 'name': 'Aerosmith'}) + monkeypatch.setattr( + library_reorganize, 'get_album_tracks_for_source', + lambda *a: {'items': [ + {'id': 'a1', 'name': 'Same Old Song And Dance', 'track_number': 1}, + {'id': 'a2', 'name': 'Dream On', 'track_number': 2}, + ]}, + ) + + pp_titles = [] + + def pp(key, ctx, fp): + pp_titles.append(ctx['track_info']['name']) + ctx['_final_processed_path'] = fp + with open(fp, 'wb') as f: + f.write(b'final') + + summary = library_reorganize.reorganize_album( + album_id='alb-1', db=db, staging_root=str(staging), + resolve_file_path_fn=lambda p: p, post_process_fn=pp, + transfer_dir=str(transfer), + ) + + # Only the normal track was post-processed; the quarantined one was not. + assert pp_titles == ['Same Old Song And Dance'] + assert summary['moved'] == 1 + assert summary['skipped'] == 1 + # The quarantined file is untouched on disk. + assert deleted_file.exists() diff --git a/tests/test_reorganize_deleted_quarantine.py b/tests/test_reorganize_deleted_quarantine.py new file mode 100644 index 00000000..436901c4 --- /dev/null +++ b/tests/test_reorganize_deleted_quarantine.py @@ -0,0 +1,72 @@ +"""Reorganize must skip files in the duplicate-cleaner quarantine (#746). + +The Duplicate Cleaner moves de-duplicated files into ``/deleted/``. +If the user's media server scans the transfer folder (e.g. a ``/music`` root +holding both the library and the transfer dir), those quarantined files get real +rows in SoulSync's DB. Reorganize is purely DB-driven, so without a guard it +would move them back OUT of /deleted to the template location. + +These tests pin: + 1. ``_is_in_deleted_quarantine`` — the anchored detection, including the + false-positive guard (an album literally named "Deleted" elsewhere is kept). + 2. ``preview_album_reorganize`` — a quarantined track is surfaced as a skip, + a normal track is not (proves the planner-shared guard fires on the path + both preview AND apply run through). +""" + +from __future__ import annotations + +import os + +from core.library_reorganize import _is_in_deleted_quarantine + + +TRANSFER = os.path.join(os.sep, "music", "soulsync") + + +class TestIsInDeletedQuarantine: + def test_file_directly_in_quarantine_is_flagged(self): + p = os.path.join(TRANSFER, "deleted", "Artist", "Album", "01.flac") + assert _is_in_deleted_quarantine(p, TRANSFER) is True + + def test_file_in_normal_album_is_not_flagged(self): + p = os.path.join(TRANSFER, "Artist", "Album", "01.flac") + assert _is_in_deleted_quarantine(p, TRANSFER) is False + + def test_album_with_deleted_in_name_is_kept(self): + """Anchored to the /deleted PREFIX, not a substring — a + real album like 'Deleted Scenes' nested under an artist must NOT be + skipped.""" + p = os.path.join(TRANSFER, "Artist", "Deleted Scenes", "01.flac") + assert _is_in_deleted_quarantine(p, TRANSFER) is False + + def test_known_unavoidable_collision_is_documented(self): + """The ONE genuine ambiguity: an artist folder named exactly + 'deleted' sitting directly at the transfer root occupies the same + path as the duplicate-cleaner quarantine, so it IS treated as + quarantine. This is unavoidable (we can't tell a real 'deleted' + artist from the cleaner's dir) and accepted — pinned here so the + behavior is intentional, not a surprise. Differently-cased or + nested 'Deleted' names are safe (see the other tests).""" + collision = os.path.join(TRANSFER, "deleted", "Album", "01.flac") + assert _is_in_deleted_quarantine(collision, TRANSFER) is True + + def test_substring_not_matched(self): + """'Undeleted' / 'deleted_scenes' as a segment must not trip the + exact-segment / prefix check.""" + p = os.path.join(TRANSFER, "Undeleted", "Album", "01.flac") + assert _is_in_deleted_quarantine(p, TRANSFER) is False + + def test_no_transfer_dir_falls_back_to_segment_match(self): + p = os.path.join(os.sep, "anywhere", "deleted", "x.flac") + assert _is_in_deleted_quarantine(p, None) is True + p2 = os.path.join(os.sep, "anywhere", "Undeleted", "x.flac") + assert _is_in_deleted_quarantine(p2, None) is False + + def test_empty_path_is_safe(self): + assert _is_in_deleted_quarantine(None, TRANSFER) is False + assert _is_in_deleted_quarantine("", TRANSFER) is False + + def test_nested_subfolders_under_quarantine_flagged(self): + p = os.path.join(TRANSFER, "deleted", "a", "b", "c", "track.flac") + assert _is_in_deleted_quarantine(p, TRANSFER) is True From 443257915cbec9fc9367fa8ca29b3087640fe55e Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Sat, 30 May 2026 00:31:14 -0700 Subject: [PATCH 52/52] Path builder: validate $year, never blind-slice release_date (#745) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The $year template variable was a blind release_date[:4] slice. When something upstream poisoned release_date with a non-date value — the album NAME — that slice emitted garbage: 'Mantras (Deluxe)'[:4] -> 'Mant', so every download landed in 'Mantras (Deluxe) (Mant) [Album]/' instead of '(2026)' (Tacobell444's screenshot). Add _extract_year_from_release_date(): returns the leading 4 chars only when they're a plausible year (isdigit, 1900 < y <= 2100), else ''. Matches the guard the codebase already uses in soulid_worker._extract_year. A non-year resolves to '' and the template's existing empty-() cleanup drops it, so a poisoned release_date can never write rubbish into the path again. This is the shared post-process path builder (core/imports/paths.build_final_path_for_track) that DOWNLOADS, reorganize, and imports all route through, so the guard covers every surface at once. Defensive fix only — it stops the SYMPTOM regardless of which upstream writes the album name into release_date. Pinning that upstream needs the reporter's metadata source + the release_date value from app.log (the Soulseek + AcoustID + future-dated-album combo is the discriminator); tracked separately. Tests (tests/imports/test_import_paths.py): unit coverage for the helper (real dates kept, names/sentinels/short values rejected) + an integration test reproducing #745 — a poisoned release_date yields 'Mantras (Deluxe) [Album]' not '(Mant)' — differential-verified it produces the exact '(Mant)' folder without the fix. Positive control keeps real (2026). 395 import + reorganize tests green. --- core/imports/paths.py | 24 ++++- tests/imports/test_import_paths.py | 153 +++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 3 deletions(-) diff --git a/core/imports/paths.py b/core/imports/paths.py index bdc12616..af9619d7 100644 --- a/core/imports/paths.py +++ b/core/imports/paths.py @@ -38,6 +38,26 @@ def _get_config_manager(): return _FallbackConfig() +def _extract_year_from_release_date(release_date) -> str: + """Return a validated 4-digit year from a release_date, or '' . + + The ``$year`` template variable used to be a blind ``release_date[:4]`` + slice. When something upstream poisons ``release_date`` with a non-date + value (e.g. the album NAME — #745, where "Mantras (Deluxe)" produced a + "(Mant)" folder), that slice happily emitted garbage. Validate that the + leading 4 chars are a plausible year, matching the guard the rest of the + codebase already uses (see ``soulid_worker._extract_year``). Anything + that isn't a real year resolves to '' — the template's bracket-cleanup + then drops the empty ``()`` instead of writing rubbish into the path. + """ + if not release_date: + return "" + candidate = str(release_date)[:4] + if candidate.isdigit() and 1900 < int(candidate) <= 2100: + return candidate + return "" + + def _get_itunes_client(): try: from core.metadata_service import get_itunes_client @@ -426,9 +446,7 @@ def build_final_path_for_track(context, artist_context, album_info, file_ext): year = "" if album_context and album_context.get("release_date"): - release_date = album_context["release_date"] - if release_date and len(release_date) >= 4: - year = release_date[:4] + year = _extract_year_from_release_date(album_context["release_date"]) raw_album_type = "" if album_context: diff --git a/tests/imports/test_import_paths.py b/tests/imports/test_import_paths.py index 899bfec6..ec94881d 100644 --- a/tests/imports/test_import_paths.py +++ b/tests/imports/test_import_paths.py @@ -372,3 +372,156 @@ def test_build_final_path_for_track_with_cdnum_template_skips_disc_folder(monkey ) # Verify the disc folder was not created either. assert not (tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 2").exists() + + +# ── #745: $year must validate, never blind-slice release_date ────────────── + +def test_extract_year_accepts_real_dates(): + assert import_paths._extract_year_from_release_date("2026-01-01") == "2026" + assert import_paths._extract_year_from_release_date("1999") == "1999" + assert import_paths._extract_year_from_release_date("2026") == "2026" + # Datetime-ish string — still leads with a valid year. + assert import_paths._extract_year_from_release_date("2010-12-31T00:00:00Z") == "2010" + + +def test_extract_year_rejects_non_date_values(): + # #745 exact case: release_date poisoned with the album NAME. + assert import_paths._extract_year_from_release_date("Mantras (Deluxe)") == "" + assert import_paths._extract_year_from_release_date("Mant") == "" + # Implausible / sentinel years are rejected. + assert import_paths._extract_year_from_release_date("0000") == "" + assert import_paths._extract_year_from_release_date("1800") == "" + assert import_paths._extract_year_from_release_date("9999") == "" + # Empty / None. + assert import_paths._extract_year_from_release_date("") == "" + assert import_paths._extract_year_from_release_date(None) == "" + # Fewer than 4 leading digits. + assert import_paths._extract_year_from_release_date("202") == "" + + +def test_build_final_path_drops_garbage_year_from_folder(monkeypatch, tmp_path): + """#745 reproduction: release_date carries the album NAME, not a date. + The $year slot must resolve to empty and the bracket cleanup must drop the + empty () — producing 'Album One' NOT 'Album One (Mant)'.""" + config = _Config( + { + "soulseek.transfer_path": str(tmp_path / "Transfer"), + "file_organization.enabled": True, + "file_organization.templates": { + # Template that uses $year, like the reporter's. + "album_path": "$albumartist/$album ($year) [$albumtype]/$track - $title", + "single_path": "$artist/$artist - $title", + }, + "file_organization.collab_artist_mode": "first", + "file_organization.disc_label": "Disc", + } + ) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + monkeypatch.setattr( + import_paths, "_get_album_tracks_for_source", + lambda source, album_id: {"items": [{"disc_number": 1}]}, + ) + + context = { + "artist": {"name": "Katie Pruitt"}, + "album": { + "name": "Mantras (Deluxe)", + "id": "album-1", + # POISONED: the album name landed in release_date (the #745 bug). + "release_date": "Mantras (Deluxe)", + "total_tracks": 12, + "album_type": "album", + "artists": [{"name": "Katie Pruitt"}], + }, + "track_info": { + "name": "White Lies, White Jesus And You", + "id": "track-1", + "track_number": 1, + "disc_number": 1, + "artists": [{"name": "Katie Pruitt"}], + }, + "original_search_result": { + "title": "White Lies, White Jesus And You", + "clean_title": "White Lies, White Jesus And You", + "clean_album": "Mantras (Deluxe)", + "clean_artist": "Katie Pruitt", + "artists": [{"name": "Katie Pruitt"}], + }, + "source": "deezer", + "is_album_download": False, + } + + final_path, created = import_paths.build_final_path_for_track( + context, + {"name": "Katie Pruitt"}, + {"is_album": True, "album_name": "Mantras (Deluxe)", "track_number": 1, "disc_number": 1}, + ".flac", + ) + + assert created is True + # The album folder must NOT contain "(Mant)" or any "(...)" year artifact. + album_folder = os.path.basename(os.path.dirname(final_path)) + assert "(Mant" not in album_folder + assert "()" not in album_folder + # Empty () collapses; [Album] type stays. + assert album_folder == "Mantras (Deluxe) [Album]" + + +def test_build_final_path_keeps_real_year_in_folder(monkeypatch, tmp_path): + """Positive control: a genuine release_date still produces the (YYYY) + folder — proves the guard didn't break the happy path.""" + config = _Config( + { + "soulseek.transfer_path": str(tmp_path / "Transfer"), + "file_organization.enabled": True, + "file_organization.templates": { + "album_path": "$albumartist/$album ($year) [$albumtype]/$track - $title", + "single_path": "$artist/$artist - $title", + }, + "file_organization.collab_artist_mode": "first", + "file_organization.disc_label": "Disc", + } + ) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + monkeypatch.setattr( + import_paths, "_get_album_tracks_for_source", + lambda source, album_id: {"items": [{"disc_number": 1}]}, + ) + + context = { + "artist": {"name": "Katie Pruitt"}, + "album": { + "name": "Mantras (Deluxe)", + "id": "album-1", + "release_date": "2026-04-12", + "total_tracks": 12, + "album_type": "album", + "artists": [{"name": "Katie Pruitt"}], + }, + "track_info": { + "name": "White Lies, White Jesus And You", + "id": "track-1", + "track_number": 1, + "disc_number": 1, + "artists": [{"name": "Katie Pruitt"}], + }, + "original_search_result": { + "title": "White Lies, White Jesus And You", + "clean_title": "White Lies, White Jesus And You", + "clean_album": "Mantras (Deluxe)", + "clean_artist": "Katie Pruitt", + "artists": [{"name": "Katie Pruitt"}], + }, + "source": "deezer", + "is_album_download": False, + } + + final_path, _ = import_paths.build_final_path_for_track( + context, + {"name": "Katie Pruitt"}, + {"is_album": True, "album_name": "Mantras (Deluxe)", "track_number": 1, "disc_number": 1}, + ".flac", + ) + + album_folder = os.path.basename(os.path.dirname(final_path)) + assert album_folder == "Mantras (Deluxe) (2026) [Album]"